diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 9d43b18d23d5c..07fade3ecd500 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -23,5 +23,6 @@ jobs: close-pr-message: "This PR was closed due to lack of activity. Feel free to reopen if it's still relevant." stale-pr-label: stale remove-pr-stale-when-updated: true + exempt-pr-labels: 'waiting' operations-per-run: 250 repo-token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} diff --git a/ee/api/test/__snapshots__/test_time_to_see_data.ambr b/ee/api/test/__snapshots__/test_time_to_see_data.ambr index 9458103ac1e18..48bec559c2d19 100644 --- a/ee/api/test/__snapshots__/test_time_to_see_data.ambr +++ b/ee/api/test/__snapshots__/test_time_to_see_data.ambr @@ -17,6 +17,7 @@ "user": { "distinct_id": "", "first_name": "", + "last_name": "", "email": "", "is_email_verified": false } diff --git a/ee/billing/quota_limiting.py b/ee/billing/quota_limiting.py index ef3e12a421575..0809266b1db64 100644 --- a/ee/billing/quota_limiting.py +++ b/ee/billing/quota_limiting.py @@ -147,12 +147,12 @@ def set_org_usage_summary( new_usage = copy.deepcopy(new_usage) for field in ["events", "recordings", "rows_synced"]: - resource_usage = new_usage[field] # type: ignore + resource_usage = new_usage.get(field, {"limit": None, "usage": 0, "todays_usage": 0}) if not resource_usage: continue if todays_usage: - resource_usage["todays_usage"] = todays_usage[field] # type: ignore + resource_usage["todays_usage"] = todays_usage.get(field, 0) else: # TRICKY: If we are not explictly setting todays_usage, we want to reset it to 0 IF the incoming new_usage is different if (organization.usage or {}).get(field, {}).get("usage") != resource_usage.get("usage"): diff --git a/ee/clickhouse/queries/funnels/test/__snapshots__/test_funnel.ambr b/ee/clickhouse/queries/funnels/test/__snapshots__/test_funnel.ambr index da971a756ef1d..bb14426e16441 100644 --- a/ee/clickhouse/queries/funnels/test/__snapshots__/test_funnel.ambr +++ b/ee/clickhouse/queries/funnels/test/__snapshots__/test_funnel.ambr @@ -1,28 +1,26 @@ # name: TestFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - AND (NOT has([''], "$group_0")) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (NOT has([''], "$group_0")) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events.1 @@ -138,34 +136,32 @@ # name: TestFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events_poe_v2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT OUTER JOIN - (SELECT argMax(override_person_id, version) as person_id, - old_person_id - FROM person_overrides - WHERE team_id = 2 - GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT OUTER JOIN + (SELECT argMax(override_person_id, version) as person_id, + old_person_id + FROM person_overrides + WHERE team_id = 2 + GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - AND (NOT has([''], "$group_0")) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (NOT has([''], "$group_0")) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events_poe_v2.1 @@ -287,26 +283,24 @@ # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.1 @@ -427,26 +421,24 @@ # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.3 @@ -564,26 +556,24 @@ # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.4 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.5 @@ -701,26 +691,24 @@ # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.6 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.7 @@ -838,26 +826,24 @@ # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.8 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelGroupBreakdown.test_funnel_breakdown_group.9 @@ -975,28 +961,26 @@ # name: TestStrictFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - AND (NOT has([''], "$group_0")) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (NOT has([''], "$group_0")) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestStrictFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events.1 @@ -1112,34 +1096,32 @@ # name: TestStrictFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events_poe_v2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT OUTER JOIN - (SELECT argMax(override_person_id, version) as person_id, - old_person_id - FROM person_overrides - WHERE team_id = 2 - GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT OUTER JOIN + (SELECT argMax(override_person_id, version) as person_id, + old_person_id + FROM person_overrides WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - AND (NOT has([''], "$group_0")) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (NOT has([''], "$group_0")) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestStrictFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events_poe_v2.1 @@ -1261,26 +1243,24 @@ # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.1 @@ -1401,26 +1381,24 @@ # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.3 @@ -1516,26 +1494,24 @@ # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.4 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.5 @@ -1631,26 +1607,24 @@ # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.6 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.7 @@ -1746,26 +1720,24 @@ # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.8 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestStrictFunnelGroupBreakdown.test_funnel_breakdown_group.9 @@ -1861,28 +1833,26 @@ # name: TestUnorderedFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - AND (NOT has([''], "$group_0")) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (NOT has([''], "$group_0")) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events.1 @@ -1998,34 +1968,32 @@ # name: TestUnorderedFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events_poe_v2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT OUTER JOIN - (SELECT argMax(override_person_id, version) as person_id, - old_person_id - FROM person_overrides - WHERE team_id = 2 - GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT OUTER JOIN + (SELECT argMax(override_person_id, version) as person_id, + old_person_id + FROM person_overrides WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - AND (NOT has([''], "$group_0")) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (NOT has([''], "$group_0")) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events_poe_v2.1 @@ -2147,26 +2115,24 @@ # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.1 @@ -2287,76 +2253,70 @@ # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.10 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.11 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.12 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.13 @@ -2571,76 +2531,70 @@ # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.14 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.15 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.16 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.17 @@ -2855,76 +2809,70 @@ # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.3 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.4 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.5 @@ -3139,76 +3087,70 @@ # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.6 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.7 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.8 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event IN ['buy', 'play movie', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestUnorderedFunnelGroupBreakdown.test_funnel_breakdown_group.9 diff --git a/ee/clickhouse/queries/test/__snapshots__/test_breakdown_props.ambr b/ee/clickhouse/queries/test/__snapshots__/test_breakdown_props.ambr index 3b7b79a3b02c6..415ed17869d25 100644 --- a/ee/clickhouse/queries/test/__snapshots__/test_breakdown_props.ambr +++ b/ee/clickhouse/queries/test/__snapshots__/test_breakdown_props.ambr @@ -1,267 +1,251 @@ # name: TestBreakdownProps.test_breakdown_group_props ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - AND ((isNull(replaceRegexpAll(JSONExtractRaw(group_properties_0, 'out'), '^"|"$', '')) - OR NOT JSONHas(group_properties_0, 'out'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 5 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND ((isNull(replaceRegexpAll(JSONExtractRaw(group_properties_0, 'out'), '^"|"$', '')) + OR NOT JSONHas(group_properties_0, 'out'))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 5 + OFFSET 0 ' --- # name: TestBreakdownProps.test_breakdown_group_props.1 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - AND ((isNull(replaceRegexpAll(JSONExtractRaw(group_properties_0, 'out'), '^"|"$', '')) - OR NOT JSONHas(group_properties_0, 'out'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 5 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND ((isNull(replaceRegexpAll(JSONExtractRaw(group_properties_0, 'out'), '^"|"$', '')) + OR NOT JSONHas(group_properties_0, 'out'))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 5 + OFFSET 0 ' --- # name: TestBreakdownProps.test_breakdown_person_props ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$browser'), '^"|"$', '') AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$browser'), '^"|"$', '') AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-21 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 5 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person + WHERE team_id = 2 + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-21 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 5 + OFFSET 0 ' --- # name: TestBreakdownProps.test_breakdown_person_props_materialized ' - SELECT groupArray(value) - FROM - (SELECT "pmat_$browser" AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(pmat_$browser, version) as pmat_$browser - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT "pmat_$browser" AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(pmat_$browser, version) as pmat_$browser + FROM person WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-21 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 5 - OFFSET 0) + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-21 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 5 + OFFSET 0 ' --- # name: TestBreakdownProps.test_breakdown_person_props_with_entity_filter_and_or_props_with_partial_pushdown ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$browser'), '^"|"$', '') AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - AND id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND ((replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '') ILIKE '%test%')) ) - GROUP BY id - HAVING max(is_deleted) = 0 - AND ((replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$browser'), '^"|"$', '') ILIKE '%test%')) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$browser'), '^"|"$', '') AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-21 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - AND ((has(['test2'], replaceRegexpAll(JSONExtractRaw(person_props, '$os'), '^"|"$', '')) - OR has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', '')))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 5 - OFFSET 0) + AND id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND ((replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '') ILIKE '%test%')) ) + GROUP BY id + HAVING max(is_deleted) = 0 + AND ((replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$browser'), '^"|"$', '') ILIKE '%test%')) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-21 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + AND ((has(['test2'], replaceRegexpAll(JSONExtractRaw(person_props, '$os'), '^"|"$', '')) + OR has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', '')))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 5 + OFFSET 0 ' --- # name: TestBreakdownProps.test_breakdown_session_props ' - SELECT groupArray(value) - FROM - (SELECT sessions.session_duration AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT sessions.session_duration AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestBreakdownProps.test_breakdown_with_math_property_session ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$browser'), '^"|"$', '') AS value, - sum(session_duration) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$browser'), '^"|"$', '') AS value, + sum(session_duration) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestBreakdownProps.test_breakdown_with_math_property_session.1 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$browser'), '^"|"$', '') AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$browser'), '^"|"$', '') AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr index 2d1aa22e3bed4..5fa656c60136d 100644 --- a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr @@ -13,19 +13,17 @@ # name: ClickhouseTestExperimentSecondaryResults.test_basic_secondary_metric_results.1 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestExperimentSecondaryResults.test_basic_secondary_metric_results.2 @@ -51,13 +49,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['control', 'test', 'ablahebf', ''] as breakdown_value) ARRAY + (SELECT ['control', 'test', 'ablahebf', '$$_posthog_breakdown_null_$$'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), (['control', 'test', 'ablahebf', '']), (['control', 'test', 'ablahebf', '']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['control', 'test', 'ablahebf', '$$_posthog_breakdown_null_$$']), (['control', 'test', 'ablahebf', '$$_posthog_breakdown_null_$$']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview' @@ -76,19 +74,17 @@ # name: ClickhouseTestExperimentSecondaryResults.test_basic_secondary_metric_results.3 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['$pageleave_funnel', '$pageview_funnel'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['$pageleave_funnel', '$pageview_funnel'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestExperimentSecondaryResults.test_basic_secondary_metric_results.4 diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr index 760b3a950715c..9738ec2e3a988 100644 --- a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr @@ -1,19 +1,17 @@ # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['$pageleave', '$pageview'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['$pageleave', '$pageview'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results.1 @@ -205,19 +203,17 @@ # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_and_events_out_of_time_range_timezones ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['$pageleave', '$pageview'] - AND toTimeZone(timestamp, 'Europe/Amsterdam') >= toDateTime('2020-01-01 14:20:21', 'Europe/Amsterdam') - AND toTimeZone(timestamp, 'Europe/Amsterdam') <= toDateTime('2020-01-06 10:00:00', 'Europe/Amsterdam') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['$pageleave', '$pageview'] + AND toTimeZone(timestamp, 'Europe/Amsterdam') >= toDateTime('2020-01-01 14:20:21', 'Europe/Amsterdam') + AND toTimeZone(timestamp, 'Europe/Amsterdam') <= toDateTime('2020-01-06 10:00:00', 'Europe/Amsterdam') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_and_events_out_of_time_range_timezones.1 @@ -409,19 +405,17 @@ # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_for_three_test_variants ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['$pageleave', '$pageview'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['$pageleave', '$pageview'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_for_three_test_variants.1 @@ -613,19 +607,17 @@ # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_with_hogql_aggregation ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['$pageleave', '$pageview'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['$pageleave', '$pageview'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_with_hogql_aggregation.1 @@ -817,20 +809,18 @@ # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results.1 @@ -862,7 +852,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), (['test', 'control']), (['test', 'control']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['test', 'control']), (['test', 'control']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview' @@ -882,21 +872,19 @@ # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results.2 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$feature_flag_called' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) - AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$feature_flag_called' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) + AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results.3 @@ -934,7 +922,7 @@ min(timestamp) as timestamp, breakdown_value FROM - (SELECT pdi.person_id as person_id, timestamp, transform(replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', ''), (['control', 'test']), (['control', 'test']), '$$_posthog_breakdown_other_$$') as breakdown_value + (SELECT pdi.person_id as person_id, timestamp, transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['control', 'test']), (['control', 'test']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -1113,20 +1101,18 @@ # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_for_three_test_variants ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview1' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - AND (has(['control', 'test_1', 'test_2', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview1' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND (has(['control', 'test_1', 'test_2', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_for_three_test_variants.1 @@ -1158,7 +1144,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), (['control', 'test_1', 'test_2']), (['control', 'test_1', 'test_2']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['control', 'test_1', 'test_2']), (['control', 'test_1', 'test_2']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview1' @@ -1178,21 +1164,19 @@ # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_for_three_test_variants.2 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$feature_flag_called' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - AND (has(['control', 'test_1', 'test_2', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) - AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$feature_flag_called' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND (has(['control', 'test_1', 'test_2', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) + AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_for_three_test_variants.3 @@ -1302,20 +1286,18 @@ # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2020-01-01 02:10:00', 'US/Pacific') - AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2020-01-06 07:00:00', 'US/Pacific') - AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2020-01-01 02:10:00', 'US/Pacific') + AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2020-01-06 07:00:00', 'US/Pacific') + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone.1 @@ -1347,7 +1329,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'US/Pacific')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), (['test', 'control']), (['test', 'control']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['test', 'control']), (['test', 'control']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview' @@ -1367,21 +1349,19 @@ # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone.2 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$feature_flag_called' - AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2020-01-01 02:10:00', 'US/Pacific') - AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2020-01-06 07:00:00', 'US/Pacific') - AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) - AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$feature_flag_called' + AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2020-01-01 02:10:00', 'US/Pacific') + AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2020-01-06 07:00:00', 'US/Pacific') + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) + AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone.3 @@ -1419,7 +1399,7 @@ min(timestamp) as timestamp, breakdown_value FROM - (SELECT pdi.person_id as person_id, timestamp, transform(replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', ''), (['control', 'test']), (['control', 'test']), '$$_posthog_breakdown_other_$$') as breakdown_value + (SELECT pdi.person_id as person_id, timestamp, transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['control', 'test']), (['control', 'test']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -1598,21 +1578,19 @@ # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - AND ((has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) - AND (ifNull(ilike(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'hogql'), ''), 'null'), '^"|"$', ''), 'true'), 0))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND ((has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) + AND (ifNull(ilike(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'hogql'), ''), 'null'), '^"|"$', ''), 'true'), 0))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter.1 @@ -1644,7 +1622,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), (['test', 'control']), (['test', 'control']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['test', 'control']), (['test', 'control']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview' @@ -1665,21 +1643,19 @@ # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter.2 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$feature_flag_called' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') - AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) - AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$feature_flag_called' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) + AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter.3 @@ -1717,7 +1693,7 @@ min(timestamp) as timestamp, breakdown_value FROM - (SELECT pdi.person_id as person_id, timestamp, transform(replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', ''), (['control', 'test']), (['control', 'test']), '$$_posthog_breakdown_other_$$') as breakdown_value + (SELECT pdi.person_id as person_id, timestamp, transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['control', 'test']), (['control', 'test']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_trends.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_trends.ambr index e8aa10d92f399..3b4b040a8ebd5 100644 --- a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_trends.ambr +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_trends.ambr @@ -216,19 +216,17 @@ # name: ClickhouseTestTrends.test_insight_trends_cumulative.10 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrends.test_insight_trends_cumulative.11 @@ -266,7 +264,7 @@ min(timestamp) as timestamp, breakdown_value FROM - (SELECT pdi.person_id as person_id, timestamp, transform(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), (['val', 'notval']), (['val', 'notval']), '$$_posthog_breakdown_other_$$') as breakdown_value + (SELECT pdi.person_id as person_id, timestamp, transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['val', 'notval']), (['val', 'notval']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -391,19 +389,17 @@ # name: ClickhouseTestTrends.test_insight_trends_cumulative.4 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrends.test_insight_trends_cumulative.5 @@ -435,7 +431,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), (['val', 'notval']), (['val', 'notval']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['val', 'notval']), (['val', 'notval']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview' @@ -485,40 +481,38 @@ # name: ClickhouseTestTrends.test_insight_trends_cumulative.7 ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, - count(DISTINCT pdi.person_id) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id - FROM person - WHERE team_id = 2 - AND id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''))) - AND (has(['some_val'], replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '')))) ) - GROUP BY id - HAVING max(is_deleted) = 0 - AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'key'), '^"|"$', ''))) - AND (has(['some_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'key'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, + count(DISTINCT pdi.person_id) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id + FROM person + WHERE team_id = 2 + AND id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''))) + AND (has(['some_val'], replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '')))) ) + GROUP BY id + HAVING max(is_deleted) = 0 + AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'key'), '^"|"$', ''))) + AND (has(['some_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'key'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: ClickhouseTestTrends.test_insight_trends_cumulative.8 @@ -565,7 +559,7 @@ CROSS JOIN (SELECT toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) AS timestamp, pdi.person_id AS person_id, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), (['val', 'notval']), (['val', 'notval']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['val', 'notval']), (['val', 'notval']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e INNER JOIN (SELECT distinct_id, diff --git a/ee/frontend/mobile-replay/__snapshots__/transform.test.ts.snap b/ee/frontend/mobile-replay/__snapshots__/transform.test.ts.snap index 195c8f7da943f..d9b55d12fbd4a 100644 --- a/ee/frontend/mobile-replay/__snapshots__/transform.test.ts.snap +++ b/ee/frontend/mobile-replay/__snapshots__/transform.test.ts.snap @@ -612,6 +612,25 @@ exports[`replay/transform transform inputs buttons with nested elements 1`] = ` } `; +exports[`replay/transform transform inputs closed keyboard custom event 1`] = ` +{ + "data": { + "adds": [], + "attributes": [], + "removes": [ + { + "id": 6, + "parentId": 5, + }, + ], + "source": 0, + "texts": [], + }, + "timestamp": 1, + "type": 3, +} +`; + exports[`replay/transform transform inputs input - $inputType - hello 1`] = ` { "data": { @@ -648,7 +667,7 @@ exports[`replay/transform transform inputs input - $inputType - hello 1`] = ` { "attributes": {}, "childNodes": [], - "id": 222, + "id": 223, "tagName": "div", "type": 2, }, @@ -715,7 +734,7 @@ exports[`replay/transform transform inputs input - button - click me 1`] = ` }, "childNodes": [ { - "id": 214, + "id": 215, "textContent": "click me", "type": 3, }, @@ -725,7 +744,7 @@ exports[`replay/transform transform inputs input - button - click me 1`] = ` "type": 2, }, ], - "id": 213, + "id": 214, "tagName": "div", "type": 2, }, @@ -802,17 +821,17 @@ exports[`replay/transform transform inputs input - checkbox - $value 1`] = ` "type": 2, }, { - "id": 182, + "id": 183, "textContent": "first", "type": 3, }, ], - "id": 183, + "id": 184, "tagName": "label", "type": 2, }, ], - "id": 181, + "id": 182, "tagName": "div", "type": 2, }, @@ -888,17 +907,17 @@ exports[`replay/transform transform inputs input - checkbox - $value 2`] = ` "type": 2, }, { - "id": 185, + "id": 186, "textContent": "second", "type": 3, }, ], - "id": 186, + "id": 187, "tagName": "label", "type": 2, }, ], - "id": 184, + "id": 185, "tagName": "div", "type": 2, }, @@ -976,17 +995,17 @@ exports[`replay/transform transform inputs input - checkbox - $value 3`] = ` "type": 2, }, { - "id": 188, + "id": 189, "textContent": "third", "type": 3, }, ], - "id": 189, + "id": 190, "tagName": "label", "type": 2, }, ], - "id": 187, + "id": 188, "tagName": "div", "type": 2, }, @@ -1058,7 +1077,7 @@ exports[`replay/transform transform inputs input - checkbox - $value 4`] = ` "type": 2, }, ], - "id": 190, + "id": 191, "tagName": "div", "type": 2, }, @@ -1130,7 +1149,7 @@ exports[`replay/transform transform inputs input - email - $value 1`] = ` "type": 2, }, ], - "id": 174, + "id": 175, "tagName": "div", "type": 2, }, @@ -1202,7 +1221,7 @@ exports[`replay/transform transform inputs input - number - $value 1`] = ` "type": 2, }, ], - "id": 175, + "id": 176, "tagName": "div", "type": 2, }, @@ -1274,7 +1293,7 @@ exports[`replay/transform transform inputs input - password - $value 1`] = ` "type": 2, }, ], - "id": 173, + "id": 174, "tagName": "div", "type": 2, }, @@ -1350,17 +1369,17 @@ exports[`replay/transform transform inputs input - progress - $value 1`] = ` }, "childNodes": [ { - "id": 225, + "id": 226, "textContent": "@keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } }", "type": 3, }, ], - "id": 224, + "id": 225, "tagName": "style", "type": 2, }, ], - "id": 226, + "id": 227, "tagName": "div", "type": 2, }, @@ -1370,7 +1389,7 @@ exports[`replay/transform transform inputs input - progress - $value 1`] = ` "type": 2, }, ], - "id": 223, + "id": 224, "tagName": "div", "type": 2, }, @@ -1443,7 +1462,7 @@ exports[`replay/transform transform inputs input - progress - $value 2`] = ` "type": 2, }, ], - "id": 227, + "id": 228, "tagName": "div", "type": 2, }, @@ -1516,7 +1535,7 @@ exports[`replay/transform transform inputs input - progress - 0.75 1`] = ` "type": 2, }, ], - "id": 228, + "id": 229, "tagName": "div", "type": 2, }, @@ -1589,7 +1608,7 @@ exports[`replay/transform transform inputs input - progress - 0.75 2`] = ` "type": 2, }, ], - "id": 229, + "id": 230, "tagName": "div", "type": 2, }, @@ -1661,7 +1680,7 @@ exports[`replay/transform transform inputs input - search - $value 1`] = ` "type": 2, }, ], - "id": 176, + "id": 177, "tagName": "div", "type": 2, }, @@ -1734,12 +1753,12 @@ exports[`replay/transform transform inputs input - select - hello 1`] = ` }, "childNodes": [ { - "id": 219, + "id": 220, "textContent": "hello", "type": 3, }, ], - "id": 218, + "id": 219, "tagName": "option", "type": 2, }, @@ -1747,12 +1766,12 @@ exports[`replay/transform transform inputs input - select - hello 1`] = ` "attributes": {}, "childNodes": [ { - "id": 221, + "id": 222, "textContent": "world", "type": 3, }, ], - "id": 220, + "id": 221, "tagName": "option", "type": 2, }, @@ -1762,7 +1781,7 @@ exports[`replay/transform transform inputs input - select - hello 1`] = ` "type": 2, }, ], - "id": 217, + "id": 218, "tagName": "div", "type": 2, }, @@ -1835,7 +1854,7 @@ exports[`replay/transform transform inputs input - tel - $value 1`] = ` "type": 2, }, ], - "id": 177, + "id": 178, "tagName": "div", "type": 2, }, @@ -1907,7 +1926,7 @@ exports[`replay/transform transform inputs input - text - $value 1`] = ` "type": 2, }, ], - "id": 172, + "id": 173, "tagName": "div", "type": 2, }, @@ -1979,7 +1998,7 @@ exports[`replay/transform transform inputs input - text - hello 1`] = ` "type": 2, }, ], - "id": 171, + "id": 172, "tagName": "div", "type": 2, }, @@ -2051,7 +2070,7 @@ exports[`replay/transform transform inputs input - textArea - $value 1`] = ` "type": 2, }, ], - "id": 216, + "id": 217, "tagName": "div", "type": 2, }, @@ -2123,7 +2142,7 @@ exports[`replay/transform transform inputs input - textArea - hello 1`] = ` "type": 2, }, ], - "id": 215, + "id": 216, "tagName": "div", "type": 2, }, @@ -2189,7 +2208,7 @@ exports[`replay/transform transform inputs input - toggle - $value 1`] = ` }, "childNodes": [ { - "id": 195, + "id": 196, "textContent": "first", "type": 3, }, @@ -2209,7 +2228,7 @@ exports[`replay/transform transform inputs input - toggle - $value 1`] = ` "style": "position:absolute;top:33%;left:5%;display:inline-block;width:75%;height:33%;background-color:#1d4aff;opacity: 0.2;border-radius:7.5%;", }, "childNodes": [], - "id": 193, + "id": 194, "tagName": "div", "type": 2, }, @@ -2219,12 +2238,12 @@ exports[`replay/transform transform inputs input - toggle - $value 1`] = ` "style": "position:absolute;top:1.5%;right:5%;display:flex;align-items:center;justify-content:center;width:40%;height:75%;cursor:inherit;background-color:#1d4aff;border:2px solid #1d4aff;border-radius:50%;", }, "childNodes": [], - "id": 194, + "id": 195, "tagName": "div", "type": 2, }, ], - "id": 192, + "id": 193, "tagName": "div", "type": 2, }, @@ -2234,12 +2253,12 @@ exports[`replay/transform transform inputs input - toggle - $value 1`] = ` "type": 2, }, ], - "id": 196, + "id": 197, "tagName": "label", "type": 2, }, ], - "id": 191, + "id": 192, "tagName": "div", "type": 2, }, @@ -2305,7 +2324,7 @@ exports[`replay/transform transform inputs input - toggle - $value 2`] = ` }, "childNodes": [ { - "id": 201, + "id": 202, "textContent": "second", "type": 3, }, @@ -2325,7 +2344,7 @@ exports[`replay/transform transform inputs input - toggle - $value 2`] = ` "style": "position:absolute;top:33%;left:5%;display:inline-block;width:75%;height:33%;background-color:#f3f4ef;opacity: 0.2;border-radius:7.5%;", }, "childNodes": [], - "id": 199, + "id": 200, "tagName": "div", "type": 2, }, @@ -2335,12 +2354,12 @@ exports[`replay/transform transform inputs input - toggle - $value 2`] = ` "style": "position:absolute;top:1.5%;left:5%;display:flex;align-items:center;justify-content:center;width:40%;height:75%;cursor:inherit;background-color:#f3f4ef;border:2px solid #f3f4ef;border-radius:50%;", }, "childNodes": [], - "id": 200, + "id": 201, "tagName": "div", "type": 2, }, ], - "id": 198, + "id": 199, "tagName": "div", "type": 2, }, @@ -2350,12 +2369,12 @@ exports[`replay/transform transform inputs input - toggle - $value 2`] = ` "type": 2, }, ], - "id": 202, + "id": 203, "tagName": "label", "type": 2, }, ], - "id": 197, + "id": 198, "tagName": "div", "type": 2, }, @@ -2421,7 +2440,7 @@ exports[`replay/transform transform inputs input - toggle - $value 3`] = ` }, "childNodes": [ { - "id": 207, + "id": 208, "textContent": "third", "type": 3, }, @@ -2441,7 +2460,7 @@ exports[`replay/transform transform inputs input - toggle - $value 3`] = ` "style": "position:absolute;top:33%;left:5%;display:inline-block;width:75%;height:33%;background-color:#1d4aff;opacity: 0.2;border-radius:7.5%;", }, "childNodes": [], - "id": 205, + "id": 206, "tagName": "div", "type": 2, }, @@ -2451,12 +2470,12 @@ exports[`replay/transform transform inputs input - toggle - $value 3`] = ` "style": "position:absolute;top:1.5%;right:5%;display:flex;align-items:center;justify-content:center;width:40%;height:75%;cursor:inherit;background-color:#1d4aff;border:2px solid #1d4aff;border-radius:50%;", }, "childNodes": [], - "id": 206, + "id": 207, "tagName": "div", "type": 2, }, ], - "id": 204, + "id": 205, "tagName": "div", "type": 2, }, @@ -2466,12 +2485,12 @@ exports[`replay/transform transform inputs input - toggle - $value 3`] = ` "type": 2, }, ], - "id": 208, + "id": 209, "tagName": "label", "type": 2, }, ], - "id": 203, + "id": 204, "tagName": "div", "type": 2, }, @@ -2547,7 +2566,7 @@ exports[`replay/transform transform inputs input - toggle - $value 4`] = ` "style": "position:absolute;top:33%;left:5%;display:inline-block;width:75%;height:33%;background-color:#1d4aff;opacity: 0.2;border-radius:7.5%;", }, "childNodes": [], - "id": 211, + "id": 212, "tagName": "div", "type": 2, }, @@ -2557,12 +2576,12 @@ exports[`replay/transform transform inputs input - toggle - $value 4`] = ` "style": "position:absolute;top:1.5%;right:5%;display:flex;align-items:center;justify-content:center;width:40%;height:75%;cursor:inherit;background-color:#1d4aff;border:2px solid #1d4aff;border-radius:50%;", }, "childNodes": [], - "id": 212, + "id": 213, "tagName": "div", "type": 2, }, ], - "id": 210, + "id": 211, "tagName": "div", "type": 2, }, @@ -2572,7 +2591,7 @@ exports[`replay/transform transform inputs input - toggle - $value 4`] = ` "type": 2, }, ], - "id": 209, + "id": 210, "tagName": "div", "type": 2, }, @@ -2644,7 +2663,7 @@ exports[`replay/transform transform inputs input - url - https://example.io 1`] "type": 2, }, ], - "id": 178, + "id": 179, "tagName": "div", "type": 2, }, @@ -2668,6 +2687,40 @@ exports[`replay/transform transform inputs input - url - https://example.io 1`] } `; +exports[`replay/transform transform inputs open keyboard custom event 1`] = ` +{ + "data": { + "adds": [ + { + "nextId": null, + "node": { + "attributes": { + "style": "color: #35373e;background-color: #f3f4ef;width: 100vw;height: 150px;bottom: 0;position: fixed;align-items: center;justify-content: center;display: flex;", + }, + "childNodes": [ + { + "id": 170, + "textContent": "keyboard", + "type": 3, + }, + ], + "id": 6, + "tagName": "div", + "type": 2, + }, + "parentId": 5, + }, + ], + "attributes": [], + "removes": [], + "source": 0, + "texts": [], + }, + "timestamp": 1, + "type": 3, +} +`; + exports[`replay/transform transform inputs placeholder - $inputType - $value 1`] = ` { "data": { @@ -2710,7 +2763,7 @@ exports[`replay/transform transform inputs placeholder - $inputType - $value 1`] }, "childNodes": [ { - "id": 231, + "id": 232, "textContent": "hello", "type": 3, }, @@ -2720,7 +2773,7 @@ exports[`replay/transform transform inputs placeholder - $inputType - $value 1`] "type": 2, }, ], - "id": 230, + "id": 231, "tagName": "div", "type": 2, }, @@ -3305,7 +3358,7 @@ exports[`replay/transform transform inputs radio group - $inputType - $value 1`] { "attributes": {}, "childNodes": [], - "id": 180, + "id": 181, "tagName": "div", "type": 2, }, @@ -3375,7 +3428,7 @@ exports[`replay/transform transform inputs radio_group - $inputType - $value 1`] "type": 2, }, ], - "id": 179, + "id": 180, "tagName": "div", "type": 2, }, @@ -3445,7 +3498,7 @@ exports[`replay/transform transform inputs radio_group 1`] = ` "type": 2, }, ], - "id": 170, + "id": 171, "tagName": "div", "type": 2, }, @@ -3511,7 +3564,7 @@ exports[`replay/transform transform inputs web_view - $inputType - $value 1`] = }, "childNodes": [ { - "id": 233, + "id": 234, "textContent": "web_view", "type": 3, }, @@ -3521,7 +3574,7 @@ exports[`replay/transform transform inputs web_view - $inputType - $value 1`] = "type": 2, }, ], - "id": 232, + "id": 233, "tagName": "div", "type": 2, }, diff --git a/ee/frontend/mobile-replay/index.ts b/ee/frontend/mobile-replay/index.ts index 85f2db138652f..b696832788379 100644 --- a/ee/frontend/mobile-replay/index.ts +++ b/ee/frontend/mobile-replay/index.ts @@ -5,7 +5,7 @@ import Ajv, { ErrorObject } from 'ajv' import { mobileEventWithTime } from './mobile.types' import mobileSchema from './schema/mobile/rr-mobile-schema.json' import webSchema from './schema/web/rr-web-schema.json' -import { makeFullEvent, makeIncrementalEvent, makeMetaEvent } from './transformers' +import { makeCustomEvent, makeFullEvent, makeIncrementalEvent, makeMetaEvent } from './transformers' const ajv = new Ajv({ allowUnionTypes: true, @@ -15,6 +15,7 @@ const transformers: Record eventWithTime> = { 2: makeFullEvent, 3: makeIncrementalEvent, 4: makeMetaEvent, + 5: makeCustomEvent, } const mobileSchemaValidator = ajv.compile(mobileSchema) diff --git a/ee/frontend/mobile-replay/mobile.types.ts b/ee/frontend/mobile-replay/mobile.types.ts index 959791821c5c1..992860d730918 100644 --- a/ee/frontend/mobile-replay/mobile.types.ts +++ b/ee/frontend/mobile-replay/mobile.types.ts @@ -118,12 +118,15 @@ type wireframeBase = { /** * @description x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0 */ - x: number - y: number + x?: number + y?: number /* - * @description width and height are the dimensions of the element, the only accepted units is pixels. You can omit the unit. + * @description the width dimension of the element, either '100vw' i.e. viewport width. Or a value in pixels. You can omit the unit when specifying pixels. + */ + width: number | '100vw' + /* + * @description the height dimension of the element, the only accepted units is pixels. You can omit the unit. */ - width: number height: number childWireframes?: wireframe[] type: MobileNodeType @@ -289,7 +292,34 @@ export type metaEvent = { } } -export type mobileEvent = fullSnapshotEvent | metaEvent | customEvent | incrementalSnapshotEvent +// this is a custom event _but_ rrweb only types tag as string, and we want to be more specific +export type keyboardEvent = { + type: EventType.Custom + data: { + tag: 'keyboard' + payload: + | { + open: true + styles?: MobileStyles + /** + * @description x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present then the keyboard is at the bottom of the screen + */ + x?: number + y?: number + /* + * @description the height dimension of the keyboard, the only accepted units is pixels. You can omit the unit. + */ + height: number + /* + * @description the width dimension of the keyboard, the only accepted units is pixels. You can omit the unit. If not present defaults to width of the viewport + */ + width?: number + } + | { open: false } + } +} + +export type mobileEvent = fullSnapshotEvent | metaEvent | customEvent | incrementalSnapshotEvent | keyboardEvent export type mobileEventWithTime = mobileEvent & { timestamp: number diff --git a/ee/frontend/mobile-replay/schema/mobile/rr-mobile-schema.json b/ee/frontend/mobile-replay/schema/mobile/rr-mobile-schema.json index 63151e3a48cad..bfbdd9fd4155d 100644 --- a/ee/frontend/mobile-replay/schema/mobile/rr-mobile-schema.json +++ b/ee/frontend/mobile-replay/schema/mobile/rr-mobile-schema.json @@ -119,6 +119,75 @@ }, "required": ["data", "timestamp", "type"], "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "data": { + "additionalProperties": false, + "properties": { + "payload": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "height": { + "type": "number" + }, + "open": { + "const": true, + "type": "boolean" + }, + "styles": { + "$ref": "#/definitions/MobileStyles" + }, + "width": { + "type": "number" + }, + "x": { + "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present then the keyboard is at the bottom of the screen", + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": ["open", "height"], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "open": { + "const": false, + "type": "boolean" + } + }, + "required": ["open"], + "type": "object" + } + ] + }, + "tag": { + "const": "keyboard", + "type": "string" + } + }, + "required": ["tag", "payload"], + "type": "object" + }, + "delay": { + "type": "number" + }, + "timestamp": { + "type": "number" + }, + "type": { + "$ref": "#/definitions/EventType.Custom" + } + }, + "required": ["data", "timestamp", "type"], + "type": "object" } ], "definitions": { @@ -248,7 +317,15 @@ "type": "string" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -258,7 +335,7 @@ "type": "number" } }, - "required": ["disabled", "height", "id", "inputType", "type", "width", "x", "y"], + "required": ["disabled", "height", "id", "inputType", "type", "width"], "type": "object" }, "wireframeCheckBox": { @@ -298,7 +375,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -308,7 +393,7 @@ "type": "number" } }, - "required": ["checked", "disabled", "height", "id", "inputType", "type", "width", "x", "y"], + "required": ["checked", "disabled", "height", "id", "inputType", "type", "width"], "type": "object" }, "wireframeDiv": { @@ -333,7 +418,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -343,7 +436,7 @@ "type": "number" } }, - "required": ["height", "id", "type", "width", "x", "y"], + "required": ["height", "id", "type", "width"], "type": "object" }, "wireframeImage": { @@ -372,7 +465,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -382,7 +483,7 @@ "type": "number" } }, - "required": ["height", "id", "type", "width", "x", "y"], + "required": ["height", "id", "type", "width"], "type": "object" }, "wireframeInput": { @@ -418,7 +519,15 @@ "type": "string" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -428,7 +537,7 @@ "type": "number" } }, - "required": ["disabled", "height", "id", "inputType", "type", "width", "x", "y"], + "required": ["disabled", "height", "id", "inputType", "type", "width"], "type": "object" }, "wireframeInputComponent": { @@ -484,7 +593,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -494,7 +611,7 @@ "type": "number" } }, - "required": ["height", "id", "type", "width", "x", "y"], + "required": ["height", "id", "type", "width"], "type": "object" }, "wireframeProgress": { @@ -535,7 +652,15 @@ "type": "number" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -545,7 +670,7 @@ "type": "number" } }, - "required": ["disabled", "height", "id", "inputType", "type", "width", "x", "y"], + "required": ["disabled", "height", "id", "inputType", "type", "width"], "type": "object" }, "wireframeRadio": { @@ -585,7 +710,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -595,7 +728,7 @@ "type": "number" } }, - "required": ["checked", "disabled", "height", "id", "inputType", "type", "width", "x", "y"], + "required": ["checked", "disabled", "height", "id", "inputType", "type", "width"], "type": "object" }, "wireframeRadioGroup": { @@ -620,7 +753,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -630,7 +771,7 @@ "type": "number" } }, - "required": ["height", "id", "type", "width", "x", "y"], + "required": ["height", "id", "type", "width"], "type": "object" }, "wireframeRectangle": { @@ -655,7 +796,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -665,7 +814,7 @@ "type": "number" } }, - "required": ["height", "id", "type", "width", "x", "y"], + "required": ["height", "id", "type", "width"], "type": "object" }, "wireframeSelect": { @@ -707,7 +856,15 @@ "type": "string" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -717,7 +874,7 @@ "type": "number" } }, - "required": ["disabled", "height", "id", "inputType", "type", "width", "x", "y"], + "required": ["disabled", "height", "id", "inputType", "type", "width"], "type": "object" }, "wireframeText": { @@ -745,7 +902,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -755,7 +920,7 @@ "type": "number" } }, - "required": ["height", "id", "text", "type", "width", "x", "y"], + "required": ["height", "id", "text", "type", "width"], "type": "object" }, "wireframeTextArea": { @@ -791,7 +956,15 @@ "type": "string" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -801,7 +974,7 @@ "type": "number" } }, - "required": ["disabled", "height", "id", "inputType", "type", "width", "x", "y"], + "required": ["disabled", "height", "id", "inputType", "type", "width"], "type": "object" }, "wireframeToggle": { @@ -840,7 +1013,15 @@ "$ref": "#/definitions/MobileNodeType" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -850,7 +1031,7 @@ "type": "number" } }, - "required": ["checked", "disabled", "height", "id", "inputType", "type", "width", "x", "y"], + "required": ["checked", "disabled", "height", "id", "inputType", "type", "width"], "type": "object" }, "wireframeWebView": { @@ -878,7 +1059,15 @@ "type": "string" }, "width": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "const": "100vw", + "type": "string" + } + ] }, "x": { "description": "x and y are the top left corner of the element, if they are present then the element is absolutely positioned, if they are not present this is equivalent to setting them to 0", @@ -888,7 +1077,7 @@ "type": "number" } }, - "required": ["height", "id", "type", "width", "x", "y"], + "required": ["height", "id", "type", "width"], "type": "object" } } diff --git a/ee/frontend/mobile-replay/transform.test.ts b/ee/frontend/mobile-replay/transform.test.ts index c2488eb189121..ccd5fb097c517 100644 --- a/ee/frontend/mobile-replay/transform.test.ts +++ b/ee/frontend/mobile-replay/transform.test.ts @@ -476,6 +476,26 @@ describe('replay/transform', () => { ).toMatchSnapshot() }) + test('open keyboard custom event', () => { + expect( + posthogEEModule.mobileReplay?.transformEventToWeb({ + timestamp: 1, + type: EventType.Custom, + data: { tag: 'keyboard', payload: { open: true, height: 150 } }, + }) + ).toMatchSnapshot() + }) + + test('closed keyboard custom event', () => { + expect( + posthogEEModule.mobileReplay?.transformEventToWeb({ + timestamp: 1, + type: EventType.Custom, + data: { tag: 'keyboard', payload: { open: false } }, + }) + ).toMatchSnapshot() + }) + test('radio_group', () => { expect( posthogEEModule.mobileReplay?.transformEventToWeb({ diff --git a/ee/frontend/mobile-replay/transformers.ts b/ee/frontend/mobile-replay/transformers.ts index 34fd54b82952e..b75b77cf55697 100644 --- a/ee/frontend/mobile-replay/transformers.ts +++ b/ee/frontend/mobile-replay/transformers.ts @@ -1,10 +1,21 @@ -import { EventType, fullSnapshotEvent, incrementalSnapshotEvent, metaEvent } from '@rrweb/types' +import { + addedNodeMutation, + customEvent, + EventType, + fullSnapshotEvent, + incrementalSnapshotEvent, + IncrementalSource, + metaEvent, + mutationData, +} from '@rrweb/types' +import { captureMessage } from '@sentry/react' import { attributes, elementNode, fullSnapshotEvent as MobileFullSnapshotEvent, incrementalSnapshotEvent as MobileIncrementalSnapshotEvent, + keyboardEvent, metaEvent as MobileMetaEvent, MobileNodeType, NodeType, @@ -34,6 +45,7 @@ import { makeMinimalStyles, makePositionStyles, makeStylesString, + StyleOverride, } from './wireframeStyle' const BACKGROUND = '#f3f4ef' @@ -63,7 +75,83 @@ function* ids(): Generator { // TODO this is shared for the lifetime of the page, so a very, very long-lived session could exhaust the ids const idSequence = ids() +// there are some fixed ids that we need to use for fixed elements or artificial mutations +const DOCUMENT_ID = 1 +const HTML_DOC_TYPE_ID = 2 +const HTML_ELEMENT_ID = 3 +const HEAD_ID = 4 const BODY_ID = 5 +const KEYBOARD_ID = 6 + +function isKeyboardEvent(x: unknown): x is keyboardEvent { + return ( + typeof x === 'object' && + x !== null && + 'data' in x && + typeof x.data === 'object' && + x.data !== null && + 'tag' in x.data && + x.data.tag === 'keyboard' + ) +} + +export const makeCustomEvent = ( + mobileCustomEvent: (customEvent | keyboardEvent) & { + timestamp: number + delay?: number + } +): (customEvent | incrementalSnapshotEvent) & { + timestamp: number + delay?: number +} => { + if (isKeyboardEvent(mobileCustomEvent)) { + // keyboard events are handled as incremental snapshots to add or remove a keyboard from the DOM + // TODO eventually we can pass something to makeIncrementalEvent here + const adds: addedNodeMutation[] = [] + const removes = [] + if (mobileCustomEvent.data.payload.open) { + const shouldAbsolutelyPosition = + _isPositiveInteger(mobileCustomEvent.data.payload.x) || + _isPositiveInteger(mobileCustomEvent.data.payload.y) + const styleOverride: StyleOverride | undefined = shouldAbsolutelyPosition ? undefined : { bottom: true } + const keyboardPlaceHolder = makePlaceholderElement( + { + id: KEYBOARD_ID, + type: 'placeholder', + label: 'keyboard', + height: mobileCustomEvent.data.payload.height, + width: _isPositiveInteger(mobileCustomEvent.data.payload.width) + ? mobileCustomEvent.data.payload.width + : '100vw', + }, + [], + styleOverride + ) + if (keyboardPlaceHolder) { + adds.push({ + parentId: BODY_ID, + nextId: null, + node: keyboardPlaceHolder, + }) + } else { + captureMessage('Failed to create keyboard placeholder', { extra: { mobileCustomEvent } }) + } + } else { + removes.push({ + parentId: BODY_ID, + id: KEYBOARD_ID, + }) + } + const mutation: mutationData = { adds, attributes: [], removes, source: IncrementalSource.Mutation, texts: [] } + return { + type: EventType.IncrementalSnapshot, + data: mutation, + timestamp: mobileCustomEvent.timestamp, + } + } else { + return mobileCustomEvent + } +} export const makeMetaEvent = ( mobileMetaEvent: MobileMetaEvent & { @@ -135,7 +223,11 @@ function makeWebViewElement(wireframe: wireframe, children: serializedNodeWithId return makePlaceholderElement(labelledWireframe, children) } -function makePlaceholderElement(wireframe: wireframe, children: serializedNodeWithId[]): serializedNodeWithId | null { +function makePlaceholderElement( + wireframe: wireframe, + children: serializedNodeWithId[], + styleOverride?: StyleOverride +): serializedNodeWithId | null { const txt = 'label' in wireframe && wireframe.label ? wireframe.label : wireframe.type || 'PLACEHOLDER' return { type: NodeType.Element, @@ -146,6 +238,7 @@ function makePlaceholderElement(wireframe: wireframe, children: serializedNodeWi horizontalAlign: 'center', backgroundColor: wireframe.style?.backgroundColor || BACKGROUND, color: wireframe.style?.color || FOREGROUND, + ...styleOverride, }), }, id: wireframe.id, @@ -744,19 +837,19 @@ export const makeFullEvent = ( name: 'html', publicId: '', systemId: '', - id: 2, + id: HTML_DOC_TYPE_ID, }, { type: NodeType.Element, tagName: 'html', attributes: { style: makeHTMLStyles() }, - id: 3, + id: HTML_ELEMENT_ID, childNodes: [ { type: NodeType.Element, tagName: 'head', attributes: {}, - id: 4, + id: HEAD_ID, childNodes: [], }, { @@ -777,7 +870,7 @@ export const makeFullEvent = ( ], }, ], - id: 1, + id: DOCUMENT_ID, }, initialOffset: { top: 0, diff --git a/ee/frontend/mobile-replay/wireframeStyle.ts b/ee/frontend/mobile-replay/wireframeStyle.ts index e475995e6ee71..fb5d41c762fb4 100644 --- a/ee/frontend/mobile-replay/wireframeStyle.ts +++ b/ee/frontend/mobile-replay/wireframeStyle.ts @@ -1,5 +1,10 @@ import { MobileStyles, wireframe, wireframeProgress } from './mobile.types' +// StyleOverride is defined here and not in the schema +// because these are overrides that the transformer is allowed to make +// not that clients are allowed to request +export type StyleOverride = MobileStyles & { bottom?: true } + function isNumber(candidate: unknown): candidate is number { return typeof candidate === 'number' } @@ -16,7 +21,7 @@ function ensureUnit(value: string | number): string { return isNumber(value) ? `${value}px` : value.replace(/px$/g, '') + 'px' } -function makeBorderStyles(wireframe: wireframe, styleOverride?: MobileStyles): string { +function makeBorderStyles(wireframe: wireframe, styleOverride?: StyleOverride): string { let styles = '' const combinedStyles = { @@ -47,30 +52,48 @@ function makeBorderStyles(wireframe: wireframe, styleOverride?: MobileStyles): s return styles } -export function makePositionStyles(wireframe: wireframe): string { +export function makeDimensionStyles(wireframe: wireframe): string { let styles = '' - if (isNumber(wireframe.width)) { + + if (wireframe.width === '100vw') { + styles += `width: 100vw;` + } else if (isNumber(wireframe.width)) { styles += `width: ${ensureUnit(wireframe.width)};` } + if (isNumber(wireframe.height)) { styles += `height: ${ensureUnit(wireframe.height)};` } - const posX = wireframe.x || 0 - const posY = wireframe.y || 0 - if (isNumber(posX) || isNumber(posY)) { + return styles +} + +export function makePositionStyles(wireframe: wireframe, styleOverride?: StyleOverride): string { + let styles = '' + + styles += makeDimensionStyles(wireframe) + + if (styleOverride?.bottom) { + styles += `bottom: 0;` styles += `position: fixed;` - if (isNumber(posX)) { - styles += `left: ${ensureUnit(posX)};` - } - if (isNumber(posY)) { - styles += `top: ${ensureUnit(posY)};` + } else { + const posX = wireframe.x || 0 + const posY = wireframe.y || 0 + if (isNumber(posX) || isNumber(posY)) { + styles += `position: fixed;` + if (isNumber(posX)) { + styles += `left: ${ensureUnit(posX)};` + } + if (isNumber(posY)) { + styles += `top: ${ensureUnit(posY)};` + } } } + return styles } -function makeLayoutStyles(wireframe: wireframe, styleOverride?: MobileStyles): string { +function makeLayoutStyles(wireframe: wireframe, styleOverride?: StyleOverride): string { let styles = '' const combinedStyles = { @@ -94,7 +117,7 @@ function makeLayoutStyles(wireframe: wireframe, styleOverride?: MobileStyles): s return styles } -function makeFontStyles(wireframe: wireframe, styleOverride?: MobileStyles): string { +function makeFontStyles(wireframe: wireframe, styleOverride?: StyleOverride): string { let styles = '' const combinedStyles = { @@ -115,7 +138,7 @@ function makeFontStyles(wireframe: wireframe, styleOverride?: MobileStyles): str return styles } -export function makeIndeterminateProgressStyles(wireframe: wireframeProgress, styleOverride?: MobileStyles): string { +export function makeIndeterminateProgressStyles(wireframe: wireframeProgress, styleOverride?: StyleOverride): string { let styles = '' const combinedStyles = { ...wireframe.style, @@ -132,7 +155,7 @@ export function makeIndeterminateProgressStyles(wireframe: wireframeProgress, st return styles } -export function makeDeterminateProgressStyles(wireframe: wireframeProgress, styleOverride?: MobileStyles): string { +export function makeDeterminateProgressStyles(wireframe: wireframeProgress, styleOverride?: StyleOverride): string { let styles = '' const combinedStyles = { ...wireframe.style, @@ -156,17 +179,17 @@ export function makeDeterminateProgressStyles(wireframe: wireframeProgress, styl /** * normally use makeStylesString instead, but sometimes you need styles without any colors applied * */ -export function makeMinimalStyles(wireframe: wireframe, styleOverride?: MobileStyles): string { +export function makeMinimalStyles(wireframe: wireframe, styleOverride?: StyleOverride): string { let styles = '' - styles += makePositionStyles(wireframe) + styles += makePositionStyles(wireframe, styleOverride) styles += makeLayoutStyles(wireframe, styleOverride) styles += makeFontStyles(wireframe, styleOverride) return styles } -export function makeColorStyles(wireframe: wireframe, styleOverride?: MobileStyles): string { +export function makeColorStyles(wireframe: wireframe, styleOverride?: StyleOverride): string { let styles = '' const combinedStyles = { @@ -186,7 +209,7 @@ export function makeColorStyles(wireframe: wireframe, styleOverride?: MobileStyl return styles } -export function makeStylesString(wireframe: wireframe, styleOverride?: MobileStyles): string { +export function makeStylesString(wireframe: wireframe, styleOverride?: StyleOverride): string { let styles = '' styles += makeColorStyles(wireframe, styleOverride) diff --git a/frontend/@posthog/lemon-ui/src/index.ts b/frontend/@posthog/lemon-ui/src/index.ts index 98bf67671d708..bd2d23a5b74ea 100644 --- a/frontend/@posthog/lemon-ui/src/index.ts +++ b/frontend/@posthog/lemon-ui/src/index.ts @@ -1,8 +1,6 @@ import '../../../src/styles/global.scss' export * from 'lib/lemon-ui/hooks' -export * from 'lib/lemon-ui/lemonToast' - export * from 'lib/lemon-ui/LemonActionableTooltip' export * from 'lib/lemon-ui/LemonBadge' export * from 'lib/lemon-ui/LemonBanner' @@ -31,6 +29,7 @@ export * from 'lib/lemon-ui/LemonTable' export * from 'lib/lemon-ui/LemonTabs' export * from 'lib/lemon-ui/LemonTag' export * from 'lib/lemon-ui/LemonTextArea' +export * from 'lib/lemon-ui/LemonToast' export * from 'lib/lemon-ui/LemonWidget' export * from 'lib/lemon-ui/Lettermark' export * from 'lib/lemon-ui/Link' diff --git a/frontend/__snapshots__/components-command-bar--actions.png b/frontend/__snapshots__/components-command-bar--actions.png index 618e7f508478b..5bfd5d633b88a 100644 Binary files a/frontend/__snapshots__/components-command-bar--actions.png and b/frontend/__snapshots__/components-command-bar--actions.png differ diff --git a/frontend/__snapshots__/components-command-bar--search--dark.png b/frontend/__snapshots__/components-command-bar--search--dark.png index 05bf95416f6e6..e30c815509d59 100644 Binary files a/frontend/__snapshots__/components-command-bar--search--dark.png and b/frontend/__snapshots__/components-command-bar--search--dark.png differ diff --git a/frontend/__snapshots__/components-command-bar--shortcuts--dark.png b/frontend/__snapshots__/components-command-bar--shortcuts--dark.png index 76944015cb2ff..cf22f0bbfd443 100644 Binary files a/frontend/__snapshots__/components-command-bar--shortcuts--dark.png and b/frontend/__snapshots__/components-command-bar--shortcuts--dark.png differ diff --git a/frontend/__snapshots__/components-command-bar--shortcuts--light.png b/frontend/__snapshots__/components-command-bar--shortcuts--light.png index 8627944da237a..723f911510e57 100644 Binary files a/frontend/__snapshots__/components-command-bar--shortcuts--light.png and b/frontend/__snapshots__/components-command-bar--shortcuts--light.png differ diff --git a/frontend/__snapshots__/components-command-bar--shortcuts.png b/frontend/__snapshots__/components-command-bar--shortcuts.png index 068cb57a254f1..57eb1cfd38813 100644 Binary files a/frontend/__snapshots__/components-command-bar--shortcuts.png and b/frontend/__snapshots__/components-command-bar--shortcuts.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-select--clearable.png b/frontend/__snapshots__/lemon-ui-lemon-select--clearable.png index 173e6083b8d7f..ddb733f7769c0 100644 Binary files a/frontend/__snapshots__/lemon-ui-lemon-select--clearable.png and b/frontend/__snapshots__/lemon-ui-lemon-select--clearable.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-select--custom-element.png b/frontend/__snapshots__/lemon-ui-lemon-select--custom-element.png index f5f49e01f6c14..fd74d1a9f72df 100644 Binary files a/frontend/__snapshots__/lemon-ui-lemon-select--custom-element.png and b/frontend/__snapshots__/lemon-ui-lemon-select--custom-element.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-select--flat.png b/frontend/__snapshots__/lemon-ui-lemon-select--flat.png index 014fa2c439173..5988b5802d517 100644 Binary files a/frontend/__snapshots__/lemon-ui-lemon-select--flat.png and b/frontend/__snapshots__/lemon-ui-lemon-select--flat.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-select--full-width.png b/frontend/__snapshots__/lemon-ui-lemon-select--full-width.png index 43c60b1213a32..fed02e9e5858e 100644 Binary files a/frontend/__snapshots__/lemon-ui-lemon-select--full-width.png and b/frontend/__snapshots__/lemon-ui-lemon-select--full-width.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-select--long-options.png b/frontend/__snapshots__/lemon-ui-lemon-select--long-options.png index b5e107be8cf4f..c2fea75877d90 100644 Binary files a/frontend/__snapshots__/lemon-ui-lemon-select--long-options.png and b/frontend/__snapshots__/lemon-ui-lemon-select--long-options.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-select--mixed-values-types.png b/frontend/__snapshots__/lemon-ui-lemon-select--mixed-values-types.png index 014fa2c439173..5988b5802d517 100644 Binary files a/frontend/__snapshots__/lemon-ui-lemon-select--mixed-values-types.png and b/frontend/__snapshots__/lemon-ui-lemon-select--mixed-values-types.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-select--nested-select.png b/frontend/__snapshots__/lemon-ui-lemon-select--nested-select.png index 014fa2c439173..5988b5802d517 100644 Binary files a/frontend/__snapshots__/lemon-ui-lemon-select--nested-select.png and b/frontend/__snapshots__/lemon-ui-lemon-select--nested-select.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-select--sectioned-options.png b/frontend/__snapshots__/lemon-ui-lemon-select--sectioned-options.png index 014fa2c439173..5988b5802d517 100644 Binary files a/frontend/__snapshots__/lemon-ui-lemon-select--sectioned-options.png and b/frontend/__snapshots__/lemon-ui-lemon-select--sectioned-options.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error--dark.png b/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error--dark.png new file mode 100644 index 0000000000000..03d8e6b457899 Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error--dark.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error--light.png b/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error--light.png new file mode 100644 index 0000000000000..1189ff1ea715e Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error--light.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error.png b/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error.png new file mode 100644 index 0000000000000..7650a39fcabb7 Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--billing-error.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types--dark.png b/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types--dark.png new file mode 100644 index 0000000000000..6e5c72e870205 Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types--dark.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types--light.png b/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types--light.png new file mode 100644 index 0000000000000..d77972acbab6a Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types--light.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types.png b/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types.png new file mode 100644 index 0000000000000..851f99b599e54 Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--toast-types.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--with-button--dark.png b/frontend/__snapshots__/lemon-ui-lemon-toast--with-button--dark.png new file mode 100644 index 0000000000000..8a4a540bc8880 Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--with-button--dark.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--with-button--light.png b/frontend/__snapshots__/lemon-ui-lemon-toast--with-button--light.png new file mode 100644 index 0000000000000..d8dc45f1d2c33 Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--with-button--light.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--with-button.png b/frontend/__snapshots__/lemon-ui-lemon-toast--with-button.png new file mode 100644 index 0000000000000..e6498725bc033 Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--with-button.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress--dark.png b/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress--dark.png new file mode 100644 index 0000000000000..ec876e2c354bb Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress--dark.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress--light.png b/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress--light.png new file mode 100644 index 0000000000000..f6dee3bda888b Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress--light.png differ diff --git a/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress.png b/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress.png new file mode 100644 index 0000000000000..8e9382795d02d Binary files /dev/null and b/frontend/__snapshots__/lemon-ui-lemon-toast--with-progress.png differ diff --git a/frontend/__snapshots__/lemon-ui-scrollable-shadows--vertical.png b/frontend/__snapshots__/lemon-ui-scrollable-shadows--vertical.png index 68ad8b2a9effb..6c099a9d15d54 100644 Binary files a/frontend/__snapshots__/lemon-ui-scrollable-shadows--vertical.png and b/frontend/__snapshots__/lemon-ui-scrollable-shadows--vertical.png differ diff --git a/frontend/__snapshots__/scenes-app-experiments--complete-funnel-experiment.png b/frontend/__snapshots__/scenes-app-experiments--complete-funnel-experiment.png index cc6dbe7d20f0d..57e239a6e6bb4 100644 Binary files a/frontend/__snapshots__/scenes-app-experiments--complete-funnel-experiment.png and b/frontend/__snapshots__/scenes-app-experiments--complete-funnel-experiment.png differ diff --git a/frontend/__snapshots__/scenes-app-experiments--experiments-list.png b/frontend/__snapshots__/scenes-app-experiments--experiments-list.png index 3342ee2a14a57..5d6749345e2a4 100644 Binary files a/frontend/__snapshots__/scenes-app-experiments--experiments-list.png and b/frontend/__snapshots__/scenes-app-experiments--experiments-list.png differ diff --git a/frontend/__snapshots__/scenes-app-experiments--running-trend-experiment.png b/frontend/__snapshots__/scenes-app-experiments--running-trend-experiment.png index 2de0013bf66f5..9d4400a99260a 100644 Binary files a/frontend/__snapshots__/scenes-app-experiments--running-trend-experiment.png and b/frontend/__snapshots__/scenes-app-experiments--running-trend-experiment.png differ diff --git a/frontend/__snapshots__/scenes-app-feature-flags--feature-flags-list.png b/frontend/__snapshots__/scenes-app-feature-flags--feature-flags-list.png index eaaba388c9c59..a01936611ef6e 100644 Binary files a/frontend/__snapshots__/scenes-app-feature-flags--feature-flags-list.png and b/frontend/__snapshots__/scenes-app-feature-flags--feature-flags-list.png differ diff --git a/frontend/__snapshots__/scenes-app-feature-flags--new-feature-flag.png b/frontend/__snapshots__/scenes-app-feature-flags--new-feature-flag.png index cb7249e0ab666..d67e7c8956499 100644 Binary files a/frontend/__snapshots__/scenes-app-feature-flags--new-feature-flag.png and b/frontend/__snapshots__/scenes-app-feature-flags--new-feature-flag.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom.png index 136e31d236554..c3c1fd9d8f2eb 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-area-breakdown-edit--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-area-breakdown-edit--webkit.png index 032441f9adaa0..7177d252228bc 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-area-breakdown-edit--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-area-breakdown-edit--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-area-breakdown-edit.png b/frontend/__snapshots__/scenes-app-insights--trends-area-breakdown-edit.png index 3ff15b4ef4079..45231cbfc15f8 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-area-breakdown-edit.png and b/frontend/__snapshots__/scenes-app-insights--trends-area-breakdown-edit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-bar-breakdown-edit--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-bar-breakdown-edit--webkit.png index c3042ba21e818..b039274181fe8 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-bar-breakdown-edit--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-bar-breakdown-edit--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-bar-breakdown-edit.png b/frontend/__snapshots__/scenes-app-insights--trends-bar-breakdown-edit.png index b125d25ed7e22..4d238f07a6aa3 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-bar-breakdown-edit.png and b/frontend/__snapshots__/scenes-app-insights--trends-bar-breakdown-edit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-line-breakdown-edit--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-line-breakdown-edit--webkit.png index 0e780666bc12e..6cb1cdcce2dba 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-line-breakdown-edit--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-line-breakdown-edit--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-line-breakdown-edit.png b/frontend/__snapshots__/scenes-app-insights--trends-line-breakdown-edit.png index 18f751cf0eab3..b667d953491c8 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-line-breakdown-edit.png and b/frontend/__snapshots__/scenes-app-insights--trends-line-breakdown-edit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-pie-breakdown-edit--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-pie-breakdown-edit--webkit.png index 98c5274496d9d..7867b6f051bb6 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-pie-breakdown-edit--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-pie-breakdown-edit--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-pie-breakdown-edit.png b/frontend/__snapshots__/scenes-app-insights--trends-pie-breakdown-edit.png index 968fb23bb9228..71a197918710c 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-pie-breakdown-edit.png and b/frontend/__snapshots__/scenes-app-insights--trends-pie-breakdown-edit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown-edit--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown-edit--webkit.png index ef61297736835..121a8264ecfe2 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown-edit--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown-edit--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown-edit.png b/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown-edit.png index bd8cb78fc7f40..f7c771b03ba85 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown-edit.png and b/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown-edit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-value-breakdown-edit--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-value-breakdown-edit--webkit.png index 0533de4693a29..10e6d25f0a574 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-value-breakdown-edit--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-value-breakdown-edit--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-value-breakdown-edit.png b/frontend/__snapshots__/scenes-app-insights--trends-value-breakdown-edit.png index c82dd5a7302c8..c4a1d5c25a4b6 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-value-breakdown-edit.png and b/frontend/__snapshots__/scenes-app-insights--trends-value-breakdown-edit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-world-map-edit--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-world-map-edit--webkit.png index 378dc8501afb5..9ff85fc35e6d2 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-world-map-edit--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-world-map-edit--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-world-map-edit.png b/frontend/__snapshots__/scenes-app-insights--trends-world-map-edit.png index 30a1dc81d26dc..2897b5ed1d2b4 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-world-map-edit.png and b/frontend/__snapshots__/scenes-app-insights--trends-world-map-edit.png differ diff --git a/frontend/__snapshots__/scenes-app-saved-insights--card-view.png b/frontend/__snapshots__/scenes-app-saved-insights--card-view.png index 3d306a9235a94..6ada9d2942660 100644 Binary files a/frontend/__snapshots__/scenes-app-saved-insights--card-view.png and b/frontend/__snapshots__/scenes-app-saved-insights--card-view.png differ diff --git a/frontend/__snapshots__/scenes-app-sidepanels--side-panel-docs.png b/frontend/__snapshots__/scenes-app-sidepanels--side-panel-docs.png index 9b49dc8cb9b3c..ea0c9a913953f 100644 Binary files a/frontend/__snapshots__/scenes-app-sidepanels--side-panel-docs.png and b/frontend/__snapshots__/scenes-app-sidepanels--side-panel-docs.png differ diff --git a/frontend/__snapshots__/scenes-app-surveys--surveys-list.png b/frontend/__snapshots__/scenes-app-surveys--surveys-list.png index 60da5cf713cb3..78da6dfa1a967 100644 Binary files a/frontend/__snapshots__/scenes-app-surveys--surveys-list.png and b/frontend/__snapshots__/scenes-app-surveys--surveys-list.png differ diff --git a/frontend/src/initKea.ts b/frontend/src/initKea.ts index 6b18da9f1c4cf..41cbda7fbf358 100644 --- a/frontend/src/initKea.ts +++ b/frontend/src/initKea.ts @@ -6,7 +6,7 @@ import { routerPlugin } from 'kea-router' import { subscriptionsPlugin } from 'kea-subscriptions' import { waitForPlugin } from 'kea-waitfor' import { windowValuesPlugin } from 'kea-window-values' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { identifierToHuman } from 'lib/utils' /* diff --git a/frontend/src/layout/navigation-3000/components/MinimalNavigation.tsx b/frontend/src/layout/navigation-3000/components/MinimalNavigation.tsx index 1161a4dadc1c8..cf0e586b6192a 100644 --- a/frontend/src/layout/navigation-3000/components/MinimalNavigation.tsx +++ b/frontend/src/layout/navigation-3000/components/MinimalNavigation.tsx @@ -51,7 +51,7 @@ export function MinimalNavigation(): JSX.Element { } + icon={} onClick={toggleSitePopover} > {user?.first_name || user?.email} diff --git a/frontend/src/layout/navigation-3000/components/Navbar.tsx b/frontend/src/layout/navigation-3000/components/Navbar.tsx index 75cc5fea0a135..6fbae5671a9c3 100644 --- a/frontend/src/layout/navigation-3000/components/Navbar.tsx +++ b/frontend/src/layout/navigation-3000/components/Navbar.tsx @@ -4,6 +4,7 @@ import { useActions, useValues } from 'kea' import { commandBarLogic } from 'lib/components/CommandBar/commandBarLogic' import { Resizer } from 'lib/components/Resizer/Resizer' import { ScrollableShadows } from 'lib/components/ScrollableShadows/ScrollableShadows' +import { IconWarning } from 'lib/lemon-ui/icons' import { Popover } from 'lib/lemon-ui/Popover' import { ProfilePicture } from 'lib/lemon-ui/ProfilePicture' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' @@ -20,7 +21,7 @@ import { NavbarButton } from './NavbarButton' export function Navbar(): JSX.Element { const { user } = useValues(userLogic) - const { isSitePopoverOpen } = useValues(navigationLogic) + const { isSitePopoverOpen, systemStatusHealthy } = useValues(navigationLogic) const { closeSitePopover, toggleSitePopover } = useActions(navigationLogic) const { isNavShown, isSidebarShown, activeNavbarItemId, navbarItems, mobileLayout } = useValues(navigation3000Logic) const { showSidebar, hideSidebar, toggleNavCollapsed, hideNavOnMobile } = useActions(navigation3000Logic) @@ -89,14 +90,24 @@ export function Navbar(): JSX.Element { to={urls.settings('project')} /> + {!systemStatusHealthy ? ( + } + identifier={Scene.Settings} + title="System issue!" + to={urls.instanceStatus()} + /> + ) : null} + } visible={isSitePopoverOpen} onClickOutside={closeSitePopover} placement="right-end" + className="min-w-70" > } + icon={} identifier="me" title={`Hi${user?.first_name ? `, ${user?.first_name}` : ''}!`} shortTitle={user?.first_name || user?.email} diff --git a/frontend/src/layout/navigation-3000/sidepanel/SidePanel.tsx b/frontend/src/layout/navigation-3000/sidepanel/SidePanel.tsx index 5c3057d8239ed..2a0fb240d23f9 100644 --- a/frontend/src/layout/navigation-3000/sidepanel/SidePanel.tsx +++ b/frontend/src/layout/navigation-3000/sidepanel/SidePanel.tsx @@ -1,15 +1,6 @@ import './SidePanel.scss' -import { - IconConfetti, - IconEllipsis, - IconFeatures, - IconGear, - IconInfo, - IconNotebook, - IconNotification, - IconSupport, -} from '@posthog/icons' +import { IconConfetti, IconEllipsis, IconFeatures, IconGear, IconInfo, IconNotebook, IconSupport } from '@posthog/icons' import { LemonButton, LemonMenu, LemonMenuItems } from '@posthog/lemon-ui' import clsx from 'clsx' import { useActions, useValues } from 'kea' @@ -20,7 +11,7 @@ import { NotebookPanel } from 'scenes/notebooks/NotebookPanel/NotebookPanel' import { SidePanelTab } from '~/types' -import { SidePanelActivity } from './panels/activity/SidePanelActivity' +import { SidePanelActivity, SidePanelActivityIcon } from './panels/activity/SidePanelActivity' import { SidePanelActivation, SidePanelActivationIcon } from './panels/SidePanelActivation' import { SidePanelDocs } from './panels/SidePanelDocs' import { SidePanelFeaturePreviews } from './panels/SidePanelFeaturePreviews' @@ -66,7 +57,7 @@ export const SIDE_PANEL_TABS: Record void -}): JSX.Element { - return ( -
- ({ label: name, value: name }))} - /> -
- ) -} - export const SidePanelDocs = (): JSX.Element => { const { iframeSrc, currentUrl } = useValues(sidePanelDocsLogic) const { updatePath, unmountIframe, closeSidePanel, handleExternalUrl } = useActions(sidePanelDocsLogic) @@ -131,7 +108,33 @@ export const SidePanelDocs = (): JSX.Element => { return ( <> - {menu && } + } + type="secondary" + onClick={() => { + ref.current?.contentWindow?.postMessage( + { + type: 'navigate', + url: '/docs', + }, + '*' + ) + }} + /> + + {menu && ( + ({ label: name, value: name }))} + /> + )} + +
} diff --git a/frontend/src/layout/navigation-3000/sidepanel/panels/activity/SidePanelActivity.tsx b/frontend/src/layout/navigation-3000/sidepanel/panels/activity/SidePanelActivity.tsx index 8fec932f720c9..74bb7d1e393bc 100644 --- a/frontend/src/layout/navigation-3000/sidepanel/panels/activity/SidePanelActivity.tsx +++ b/frontend/src/layout/navigation-3000/sidepanel/panels/activity/SidePanelActivity.tsx @@ -1,7 +1,9 @@ +import { IconNotification } from '@posthog/icons' import { LemonBanner, LemonButton, LemonSkeleton, LemonTabs, Link, Spinner } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { ActivityLogRow } from 'lib/components/ActivityLog/ActivityLog' import { usePageVisibility } from 'lib/hooks/usePageVisibility' +import { IconWithCount } from 'lib/lemon-ui/icons' import { useEffect, useRef } from 'react' import { urls } from 'scenes/urls' @@ -14,6 +16,16 @@ import { SidePanelPaneHeader } from '../../components/SidePanelPaneHeader' const SCROLL_TRIGGER_OFFSET = 100 +export const SidePanelActivityIcon = (props: { className?: string }): JSX.Element => { + const { unreadCount } = useValues(notificationsLogic) + + return ( + + + + ) +} + export const SidePanelActivity = (): JSX.Element => { const { hasNotifications, @@ -89,11 +101,7 @@ export const SidePanelActivity = (): JSX.Element => { {hasUnread ? (
- markAllAsRead()} - loading={importantChangesLoading} - > + markAllAsRead()}> Mark all as read
diff --git a/frontend/src/layout/navigation-3000/sidepanel/panels/sidePanelDocsLogic.ts b/frontend/src/layout/navigation-3000/sidepanel/panels/sidePanelDocsLogic.ts index fa81eaedc2f32..ae109ee3586ea 100644 --- a/frontend/src/layout/navigation-3000/sidepanel/panels/sidePanelDocsLogic.ts +++ b/frontend/src/layout/navigation-3000/sidepanel/panels/sidePanelDocsLogic.ts @@ -1,7 +1,6 @@ -import { actions, connect, kea, listeners, path, reducers, selectors } from 'kea' +import { actions, afterMount, beforeUnmount, connect, kea, listeners, path, reducers, selectors } from 'kea' import { router } from 'kea-router' - -import { SidePanelTab } from '~/types' +import { sceneLogic } from 'scenes/sceneLogic' import { sidePanelStateLogic } from '../sidePanelStateLogic' import type { sidePanelDocsLogicType } from './sidePanelDocsLogicType' @@ -26,10 +25,10 @@ export const sidePanelDocsLogic = kea([ path(['scenes', 'navigation', 'sidepanel', 'sidePanelDocsLogic']), connect({ actions: [sidePanelStateLogic, ['openSidePanel', 'closeSidePanel']], + values: [sceneLogic, ['sceneConfig']], }), actions({ - openDocsPage: (urlOrPath: string) => ({ urlOrPath }), updatePath: (path: string) => ({ path }), setInitialPath: (path: string) => ({ path }), unmountIframe: true, @@ -68,9 +67,11 @@ export const sidePanelDocsLogic = kea([ }), listeners(({ actions, values }) => ({ - openDocsPage: ({ urlOrPath }) => { - actions.setInitialPath(getPathFromUrl(urlOrPath)) - actions.openSidePanel(SidePanelTab.Docs) + openSidePanel: ({ options }) => { + if (options) { + const initialPath = getPathFromUrl(options) + actions.setInitialPath(initialPath) + } }, unmountIframe: () => { @@ -82,4 +83,14 @@ export const sidePanelDocsLogic = kea([ router.actions.push(getPathFromUrl(urlOrPath)) }, })), + + afterMount(({ actions, values }) => { + if (values.sceneConfig?.defaultDocsPath) { + actions.setInitialPath(values.sceneConfig?.defaultDocsPath) + } + }), + + beforeUnmount(({ actions, values }) => { + actions.setInitialPath(values.currentPath ?? '/docs') + }), ]) diff --git a/frontend/src/layout/navigation-3000/sidepanel/sidePanelLogic.tsx b/frontend/src/layout/navigation-3000/sidepanel/sidePanelLogic.tsx index 2039a84f6d816..c3b3b83b80177 100644 --- a/frontend/src/layout/navigation-3000/sidepanel/sidePanelLogic.tsx +++ b/frontend/src/layout/navigation-3000/sidepanel/sidePanelLogic.tsx @@ -87,10 +87,10 @@ export const sidePanelLogic = kea([ tabs.push(SidePanelTab.Support) } tabs.push(SidePanelTab.Settings) + tabs.push(SidePanelTab.Activity) if (isReady && !hasCompletedAllTasks) { tabs.push(SidePanelTab.Activation) } - tabs.push(SidePanelTab.Activity) tabs.push(SidePanelTab.FeaturePreviews) tabs.push(SidePanelTab.Welcome) @@ -99,13 +99,17 @@ export const sidePanelLogic = kea([ ], visibleTabs: [ - (s) => [s.enabledTabs, s.selectedTab, s.sidePanelOpen, s.isReady, s.hasCompletedAllTasks], - (enabledTabs, selectedTab, sidePanelOpen): SidePanelTab[] => { - return enabledTabs.filter((tab: any) => { + (s) => [s.enabledTabs, s.selectedTab, s.sidePanelOpen, s.unreadCount], + (enabledTabs, selectedTab, sidePanelOpen, unreadCount): SidePanelTab[] => { + return enabledTabs.filter((tab) => { if (tab === selectedTab && sidePanelOpen) { return true } + if (tab === SidePanelTab.Activity && unreadCount) { + return true + } + // Hide certain tabs unless they are selected if (ALWAYS_EXTRA_TABS.includes(tab)) { return false diff --git a/frontend/src/layout/navigation-3000/themeLogic.ts b/frontend/src/layout/navigation-3000/themeLogic.ts index 22b482d34662c..205e56aa099ce 100644 --- a/frontend/src/layout/navigation-3000/themeLogic.ts +++ b/frontend/src/layout/navigation-3000/themeLogic.ts @@ -9,7 +9,7 @@ import type { themeLogicType } from './themeLogicType' export const themeLogic = kea([ path(['layout', 'navigation-3000', 'themeLogic']), connect({ - values: [featureFlagLogic, ['featureFlags'], userLogic, ['user']], + values: [featureFlagLogic, ['featureFlags'], userLogic, ['themeMode']], }), actions({ syncDarkModePreference: (darkModePreference: boolean) => ({ darkModePreference }), @@ -24,8 +24,8 @@ export const themeLogic = kea([ }), selectors({ isDarkModeOn: [ - (s) => [s.user, s.darkModeSystemPreference, s.featureFlags, sceneLogic.selectors.sceneConfig], - (user, darkModeSystemPreference, featureFlags, sceneConfig) => { + (s) => [s.themeMode, s.darkModeSystemPreference, s.featureFlags, sceneLogic.selectors.sceneConfig], + (themeMode, darkModeSystemPreference, featureFlags, sceneConfig) => { // NOTE: Unauthenticated users always get the light mode until we have full support across onboarding flows if ( sceneConfig?.layout === 'plain' || @@ -34,12 +34,16 @@ export const themeLogic = kea([ ) { return false } + // Dark mode is a PostHog 3000 feature - // User-saved preference is used when set, oterwise we fall back to the system value + if (featureFlags[FEATURE_FLAGS.POSTHOG_3000] !== 'test') { + return false + } + return featureFlags[FEATURE_FLAGS.POSTHOG_3000] === 'test' - ? user?.theme_mode - ? user.theme_mode === 'dark' - : darkModeSystemPreference + ? themeMode === 'system' + ? darkModeSystemPreference + : themeMode === 'dark' : false }, ], diff --git a/frontend/src/layout/navigation/Breadcrumbs/breadcrumbsLogic.tsx b/frontend/src/layout/navigation/Breadcrumbs/breadcrumbsLogic.tsx index 2b40103a7454c..ec769563d9777 100644 --- a/frontend/src/layout/navigation/Breadcrumbs/breadcrumbsLogic.tsx +++ b/frontend/src/layout/navigation/Breadcrumbs/breadcrumbsLogic.tsx @@ -114,7 +114,7 @@ export const breadcrumbsLogic = kea([ breadcrumbs.push({ key: 'me', name: user.first_name, - symbol: , + symbol: , }) } // Instance diff --git a/frontend/src/layout/navigation/ProjectNotice.tsx b/frontend/src/layout/navigation/ProjectNotice.tsx index 4155b6682ede8..d2fdc9af280d8 100644 --- a/frontend/src/layout/navigation/ProjectNotice.tsx +++ b/frontend/src/layout/navigation/ProjectNotice.tsx @@ -22,7 +22,7 @@ interface ProjectNoticeBlueprint { export function ProjectNotice(): JSX.Element | null { const { projectNoticeVariantWithClosability } = useValues(navigationLogic) const { currentOrganization } = useValues(organizationLogic) - const { updateCurrentTeam } = useActions(userLogic) + const { updateCurrentTeam, logout } = useActions(userLogic) const { user } = useValues(userLogic) const { closeProjectNotice } = useActions(navigationLogic) const { showInviteModal } = useActions(inviteLogic) @@ -104,6 +104,11 @@ export function ProjectNotice(): JSX.Element | null { is_impersonated: { message: 'You are currently impersonating another user.', type: 'warning', + action: { + 'data-attr': 'stop-impersonation-cta', + onClick: () => logout(), + children: 'Logout', + }, }, } diff --git a/frontend/src/layout/navigation/TopBar/SitePopover.tsx b/frontend/src/layout/navigation/TopBar/SitePopover.tsx index 22796aadbf103..8a9191048b95f 100644 --- a/frontend/src/layout/navigation/TopBar/SitePopover.tsx +++ b/frontend/src/layout/navigation/TopBar/SitePopover.tsx @@ -1,5 +1,5 @@ -import { IconChevronDown, IconDay, IconFeatures, IconLaptop, IconLive, IconNight } from '@posthog/icons' -import { LemonButtonPropsBase, LemonSelect } from '@posthog/lemon-ui' +import { IconChevronDown, IconFeatures, IconLive } from '@posthog/icons' +import { LemonButtonPropsBase } from '@posthog/lemon-ui' import clsx from 'clsx' import { useActions, useValues } from 'kea' import { FlaggedFeature } from 'lib/components/FlaggedFeature' @@ -25,6 +25,7 @@ import { Tooltip } from 'lib/lemon-ui/Tooltip' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { billingLogic } from 'scenes/billing/billingLogic' import { inviteLogic } from 'scenes/settings/organization/inviteLogic' +import { ThemeSwitcher } from 'scenes/settings/user/ThemeSwitcher' import { featurePreviewsLogic } from '~/layout/FeaturePreviews/featurePreviewsLogic' import { @@ -63,22 +64,24 @@ function AccountInfo(): JSX.Element { return (
- -
- {user?.first_name} -
- {user?.email} + } + > + +
+
{user?.first_name}
+
+ {user?.email} +
-
- - } - /> - +
) } @@ -98,7 +101,7 @@ function CurrentOrganization({ organization }: { organization: OrganizationBasic onClick={closeSitePopover} >
- {organization.name} + {organization.name}
@@ -137,17 +140,17 @@ export function InviteMembersButton({ function SystemStatus(): JSX.Element { const { closeSitePopover } = useActions(navigationLogic) - const { systemStatus } = useValues(navigationLogic) + const { systemStatusHealthy } = useValues(navigationLogic) return ( : } + status={systemStatusHealthy ? 'success' : 'danger'} + icon={systemStatusHealthy ? : } fullWidth > <>
- {systemStatus ? 'All systems operational' : 'Potential system issue'} + {systemStatusHealthy ? 'All systems operational' : 'Potential system issue'}
, value: null, label: `Sync with system` }, - { icon: , value: 'light', label: 'Light mode' }, - { icon: , value: 'dark', label: 'Dark mode' }, - ]} - value={user?.theme_mode} - renderButtonContent={(leaf) => { - return ( - <> - - Color theme - {leaf ? leaf.label : 'Sync with system'} - - - ) - }} - onChange={(value) => updateUser({ theme_mode: value })} - type="tertiary" - fullWidth - dropdownPlacement="right-start" - /> - ) -} - function SignOutButton(): JSX.Element { const { logout } = useActions(userLogic) @@ -312,7 +285,7 @@ export function SitePopoverOverlay(): JSX.Element { )} - +
- - {!systemStatus && } + + {!systemStatusHealthy && }
diff --git a/frontend/src/layout/navigation/TopBar/TopBar.scss b/frontend/src/layout/navigation/TopBar/TopBar.scss index 415fb499233f3..55803cdd6ef71 100644 --- a/frontend/src/layout/navigation/TopBar/TopBar.scss +++ b/frontend/src/layout/navigation/TopBar/TopBar.scss @@ -180,7 +180,6 @@ .AccountInfo { display: flex; align-items: center; - margin: 0.5rem; } .AccountInfo__identification { diff --git a/frontend/src/layout/navigation/navigationLogic.ts b/frontend/src/layout/navigation/navigationLogic.ts index 2b11ef83f4a38..0fa4d1711161c 100644 --- a/frontend/src/layout/navigation/navigationLogic.ts +++ b/frontend/src/layout/navigation/navigationLogic.ts @@ -137,18 +137,18 @@ export const navigationLogic = kea([ !noSidebar && (mobileLayout ? isActivationSideBarShownBase && !isSideBarShownMobile : isActivationSideBarShownBase), ], - systemStatus: [ + systemStatusHealthy: [ (s) => [s.navigationStatus, preflightLogic.selectors.siteUrlMisconfigured], (status, siteUrlMisconfigured) => { - if (siteUrlMisconfigured) { - return false - } - // On cloud non staff users don't have status metrics to review if (preflightLogic.values.preflight?.cloud && !userLogic.values.user?.is_staff) { return true } + if (siteUrlMisconfigured) { + return false + } + return status.system_status_ok }, ], diff --git a/frontend/src/lib/components/ActivityLog/ActivityLog.tsx b/frontend/src/lib/components/ActivityLog/ActivityLog.tsx index 52834f7479876..fe00761d2fa49 100644 --- a/frontend/src/lib/components/ActivityLog/ActivityLog.tsx +++ b/frontend/src/lib/components/ActivityLog/ActivityLog.tsx @@ -73,9 +73,11 @@ export const ActivityLogRow = ({
diff --git a/frontend/src/lib/components/ActivityLog/humanizeActivity.tsx b/frontend/src/lib/components/ActivityLog/humanizeActivity.tsx index e1dc145a86a8b..28bdb6b4aa784 100644 --- a/frontend/src/lib/components/ActivityLog/humanizeActivity.tsx +++ b/frontend/src/lib/components/ActivityLog/humanizeActivity.tsx @@ -1,4 +1,5 @@ import { dayjs } from 'lib/dayjs' +import { fullName } from 'lib/utils' import { InsightShortId, PersonType } from '~/types' @@ -110,7 +111,7 @@ export function humanize( if (description !== null) { logLines.push({ email: logItem.user?.email, - name: logItem.user?.first_name, + name: logItem.user ? fullName(logItem.user) : undefined, isSystem: logItem.is_system, description, extendedDescription, @@ -126,5 +127,5 @@ export function userNameForLogItem(logItem: ActivityLogItem): string { if (logItem.is_system) { return 'PostHog' } - return logItem.user?.first_name ?? 'A user' + return logItem.user ? fullName(logItem.user) : 'A user' } diff --git a/frontend/src/lib/components/AddToDashboard/addToDashboardModalLogic.ts b/frontend/src/lib/components/AddToDashboard/addToDashboardModalLogic.ts index c3d333e2acd47..c32164f707c7a 100644 --- a/frontend/src/lib/components/AddToDashboard/addToDashboardModalLogic.ts +++ b/frontend/src/lib/components/AddToDashboard/addToDashboardModalLogic.ts @@ -1,7 +1,7 @@ import FuseClass from 'fuse.js' import { actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' import { router } from 'kea-router' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { newDashboardLogic } from 'scenes/dashboard/newDashboardLogic' import { insightLogic } from 'scenes/insights/insightLogic' diff --git a/frontend/src/lib/components/AnnotationsOverlay/AnnotationsOverlay.tsx b/frontend/src/lib/components/AnnotationsOverlay/AnnotationsOverlay.tsx index 3f43c79ca46d0..dd8731af9c5b4 100644 --- a/frontend/src/lib/components/AnnotationsOverlay/AnnotationsOverlay.tsx +++ b/frontend/src/lib/components/AnnotationsOverlay/AnnotationsOverlay.tsx @@ -258,8 +258,9 @@ function AnnotationCard({ annotation }: { annotation: AnnotationType }): JSX.Ele
{annotation.content}
Created by
- {' '} - +
Last modified by
- {' '} + {' '}
diff --git a/frontend/src/lib/components/CommandBar/CommandBar.tsx b/frontend/src/lib/components/CommandBar/CommandBar.tsx index 9fd65afb0478d..0e9159d59325b 100644 --- a/frontend/src/lib/components/CommandBar/CommandBar.tsx +++ b/frontend/src/lib/components/CommandBar/CommandBar.tsx @@ -29,12 +29,12 @@ const CommandBarOverlay = forwardRef(fun backdropFilter: 'blur(var(--modal-backdrop-blur))', }} > -
+
{children} diff --git a/frontend/src/lib/components/CommandBar/SearchInput.tsx b/frontend/src/lib/components/CommandBar/SearchInput.tsx index 3d79b64531e78..05ea59f7c02fa 100644 --- a/frontend/src/lib/components/CommandBar/SearchInput.tsx +++ b/frontend/src/lib/components/CommandBar/SearchInput.tsx @@ -1,4 +1,4 @@ -import { LemonInput } from '@posthog/lemon-ui' +import { LemonButton, LemonInput } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { isMac } from 'lib/utils' import { forwardRef, Ref } from 'react' @@ -11,7 +11,7 @@ import { searchBarLogic } from './searchBarLogic' export const SearchInput = forwardRef(function _SearchInput(_, ref: Ref): JSX.Element { const { currentTeam } = useValues(teamLogic) const { searchQuery } = useValues(searchBarLogic) - const { setSearchQuery } = useActions(searchBarLogic) + const { setSearchQuery, hideCommandBar } = useActions(searchBarLogic) const modifierKey = isMac() ? '⌘' : '^' const placeholder = currentTeam @@ -25,7 +25,11 @@ export const SearchInput = forwardRef(function _SearchInput(_, ref: Ref} + suffix={ + hideCommandBar()} noPadding> + + + } placeholder={placeholder} autoFocus value={searchQuery} diff --git a/frontend/src/lib/components/CommandBar/SearchResults.tsx b/frontend/src/lib/components/CommandBar/SearchResults.tsx index 9c0b6808b6cce..34416d8eaa018 100644 --- a/frontend/src/lib/components/CommandBar/SearchResults.tsx +++ b/frontend/src/lib/components/CommandBar/SearchResults.tsx @@ -1,4 +1,6 @@ +import clsx from 'clsx' import { useValues } from 'kea' +import { useResizeBreakpoints } from 'lib/hooks/useResizeObserver' import { DetectiveHog } from '../hedgehogs' import { searchBarLogic } from './searchBarLogic' @@ -9,8 +11,13 @@ export const SearchResults = (): JSX.Element => { const { combinedSearchResults, combinedSearchLoading, activeResultIndex, keyboardResultIndex } = useValues(searchBarLogic) + const { ref, size } = useResizeBreakpoints({ + 0: 'small', + 550: 'normal', + }) + return ( -
+
{!combinedSearchLoading && combinedSearchResults?.length === 0 ? (

No results

@@ -19,7 +26,12 @@ export const SearchResults = (): JSX.Element => {
) : (
-
+
{combinedSearchLoading && ( <> @@ -38,9 +50,11 @@ export const SearchResults = (): JSX.Element => { /> ))}
-
- -
+ {size !== 'small' ? ( +
+ +
+ ) : null}
)}
diff --git a/frontend/src/lib/components/CommandBar/SearchTabs.tsx b/frontend/src/lib/components/CommandBar/SearchTabs.tsx index d847de715b559..37ff41ff30a53 100644 --- a/frontend/src/lib/components/CommandBar/SearchTabs.tsx +++ b/frontend/src/lib/components/CommandBar/SearchTabs.tsx @@ -12,7 +12,7 @@ type SearchTabsProps = { export const SearchTabs = ({ inputRef }: SearchTabsProps): JSX.Element | null => { const { tabsGrouped } = useValues(searchBarLogic) return ( -
+
{Object.entries(tabsGrouped).map(([group, tabs]) => (
{group !== 'all' && ( diff --git a/frontend/src/lib/components/CommandPalette/commandPaletteLogic.tsx b/frontend/src/lib/components/CommandPalette/commandPaletteLogic.tsx index 3ccf73e40286a..f9b9157c2959b 100644 --- a/frontend/src/lib/components/CommandPalette/commandPaletteLogic.tsx +++ b/frontend/src/lib/components/CommandPalette/commandPaletteLogic.tsx @@ -689,9 +689,7 @@ export const commandPaletteLogic = kea([ }, }, { - icon: () => ( - - ), + icon: () => , display: 'Go to User settings', synonyms: ['account', 'profile'], executor: () => { @@ -927,7 +925,7 @@ export const commandPaletteLogic = kea([ icon: IconLaptop, display: 'Sync with system preferences', executor: () => { - actions.updateUser({ theme_mode: null }) + actions.updateUser({ theme_mode: 'system' }) }, }, ], diff --git a/frontend/src/lib/components/DatePicker.scss b/frontend/src/lib/components/DatePicker.scss index 21ec676273bc1..f71c348fe9062 100644 --- a/frontend/src/lib/components/DatePicker.scss +++ b/frontend/src/lib/components/DatePicker.scss @@ -27,8 +27,28 @@ background: var(--primary-3000); } + .ant-picker-time-panel .ant-picker-time-panel-column:nth-child(3)::after { + // :HACKY: fix to keep the whole am/pm section in view + display: none; + } + + .ant-picker-cell .ant-picker-cell-inner { + border-radius: var(--radius); + } + + .ant-picker-cell.ant-picker-cell-selected .ant-picker-cell-inner { + color: var(--default); + background: var(--primary); + } + + .ant-picker-cell.ant-picker-cell-today .ant-picker-cell-inner::before { + background: none; + border-color: var(--primary); + } + .ant-picker-cell:hover:not(.ant-picker-cell-in-view) .ant-picker-cell-inner, .ant-picker-cell:hover:not( + .ant-picker-cell-today, .ant-picker-cell-selected, .ant-picker-cell-range-start, .ant-picker-cell-range-end, @@ -39,4 +59,14 @@ .ant-picker-time-panel-column > li.ant-picker-time-panel-cell .ant-picker-time-panel-cell-inner:hover { background: var(--secondary-3000); } + + .ant-picker-footer .ant-btn-primary { + background: var(--primary); + border: none; + border-radius: 0.25rem; + } + + .ant-picker-footer .ant-picker-now-btn:hover { + color: var(--primary); + } } diff --git a/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx b/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx index 118327650e662..bd250bfe90adb 100644 --- a/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx +++ b/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx @@ -185,7 +185,7 @@ function Owner({ user }: { user?: UserBasicType | null }): JSX.Element { <> {user?.uuid ? (
- + {user.first_name}
) : ( diff --git a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts index 38f907c0abbee..013ecdc159c3d 100644 --- a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts +++ b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts @@ -4,7 +4,7 @@ import { loaders } from 'kea-loaders' import api from 'lib/api' import { getSingularType } from 'lib/components/DefinitionPopover/utils' import { TaxonomicDefinitionTypes, TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { capitalizeFirstLetter } from 'lib/utils' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { urls } from 'scenes/urls' diff --git a/frontend/src/lib/components/ExportButton/exporter.tsx b/frontend/src/lib/components/ExportButton/exporter.tsx index 97cff3343e00c..e1ccc83cb518f 100644 --- a/frontend/src/lib/components/ExportButton/exporter.tsx +++ b/frontend/src/lib/components/ExportButton/exporter.tsx @@ -2,7 +2,7 @@ import { AnimationType } from 'lib/animations/animations' import api from 'lib/api' import { Animation } from 'lib/components/Animation/Animation' import { dayjs } from 'lib/dayjs' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { Spinner } from 'lib/lemon-ui/Spinner/Spinner' import { delay } from 'lib/utils' import posthog from 'posthog-js' diff --git a/frontend/src/lib/components/IntervalFilter/intervalFilterLogic.ts b/frontend/src/lib/components/IntervalFilter/intervalFilterLogic.ts index e864d54edfc39..4bcf4dfa7cf2a 100644 --- a/frontend/src/lib/components/IntervalFilter/intervalFilterLogic.ts +++ b/frontend/src/lib/components/IntervalFilter/intervalFilterLogic.ts @@ -1,7 +1,7 @@ import { actions, connect, kea, key, listeners, path, props, reducers } from 'kea' import { IntervalKeyType, Intervals, intervals } from 'lib/components/IntervalFilter/intervals' import { dayjs } from 'lib/dayjs' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { dateMapping, objectsEqual } from 'lib/utils' import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import { keyForInsightLogicProps } from 'scenes/insights/sharedUtils' diff --git a/frontend/src/lib/components/MemberSelect.tsx b/frontend/src/lib/components/MemberSelect.tsx new file mode 100644 index 0000000000000..3ab1d0a793662 --- /dev/null +++ b/frontend/src/lib/components/MemberSelect.tsx @@ -0,0 +1,119 @@ +import { LemonButton, LemonButtonProps, LemonDropdown, LemonInput, ProfilePicture } from '@posthog/lemon-ui' +import { useValues } from 'kea' +import { fullName } from 'lib/utils' +import { useMemo, useState } from 'react' +import { membersLogic } from 'scenes/organization/membersLogic' + +import { UserBasicType } from '~/types' + +export type MemberSelectProps = Pick & { + defaultLabel?: string + // NOTE: Trying to cover a lot of different cases - if string we assume uuid, if number we assume id + value: UserBasicType | string | number | null + onChange: (value: UserBasicType | null) => void +} + +export function MemberSelect({ + defaultLabel = 'All users', + value, + onChange, + ...buttonProps +}: MemberSelectProps): JSX.Element { + const { meFirstMembers, membersFuse } = useValues(membersLogic) + const [showPopover, setShowPopover] = useState(false) + const [searchTerm, setSearchTerm] = useState('') + + const filteredMembers = useMemo(() => { + return searchTerm ? membersFuse.search(searchTerm).map((result) => result.item) : meFirstMembers + }, [searchTerm, meFirstMembers]) + + const selectedMember = useMemo(() => { + if (!value) { + return null + } + if (typeof value === 'string' || typeof value === 'number') { + const propToCompare = typeof value === 'string' ? 'uuid' : 'id' + return meFirstMembers.find((member) => member.user[propToCompare] === value)?.user ?? `${value}` + } + return value + }, [value, meFirstMembers]) + + const _onChange = (value: UserBasicType | null): void => { + setShowPopover(false) + onChange(value) + } + + return ( + setShowPopover(visible)} + overlay={ +
+ +
    +
  • + _onChange(null)} + > + {defaultLabel} + +
    • + + {filteredMembers.map((member) => ( +
  • + } + onClick={() => _onChange(member.user)} + > + + {fullName(member.user)} + + {meFirstMembers[0] === member && `(you)`} + + + +
    • + ))} + + {filteredMembers.length === 0 ? ( +
    + {searchTerm ? No matches : No users} +
    + ) : null} +
+
+ } + > + + {typeof selectedMember === 'string' ? ( + selectedMember + ) : selectedMember ? ( + + {fullName(selectedMember)} + {meFirstMembers[0].user.uuid === selectedMember.uuid ? ` (you)` : ''} + + ) : ( + defaultLabel + )} + + + ) +} diff --git a/frontend/src/lib/components/ObjectTags/objectTagsLogic.ts b/frontend/src/lib/components/ObjectTags/objectTagsLogic.ts index 3770b27a8a491..53420e846ad24 100644 --- a/frontend/src/lib/components/ObjectTags/objectTagsLogic.ts +++ b/frontend/src/lib/components/ObjectTags/objectTagsLogic.ts @@ -1,6 +1,6 @@ import equal from 'fast-deep-equal' import { actions, kea, key, listeners, path, props, propsChanged, reducers, selectors } from 'kea' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import type { objectTagsLogicType } from './objectTagsLogicType' diff --git a/frontend/src/lib/components/ScrollableShadows/ScrollableShadows.scss b/frontend/src/lib/components/ScrollableShadows/ScrollableShadows.scss index 30c21a8cebb4a..c5f687268294a 100644 --- a/frontend/src/lib/components/ScrollableShadows/ScrollableShadows.scss +++ b/frontend/src/lib/components/ScrollableShadows/ScrollableShadows.scss @@ -22,6 +22,8 @@ } &.ScrollableShadows--horizontal { + height: 100%; + .ScrollableShadows__inner { overflow-x: auto; } @@ -36,6 +38,8 @@ } &.ScrollableShadows--vertical { + width: 100%; + .ScrollableShadows__inner { overflow-y: auto; } diff --git a/frontend/src/lib/components/Subscriptions/subscriptionLogic.ts b/frontend/src/lib/components/Subscriptions/subscriptionLogic.ts index d6b65209c1e9b..cf673b42dbe4c 100644 --- a/frontend/src/lib/components/Subscriptions/subscriptionLogic.ts +++ b/frontend/src/lib/components/Subscriptions/subscriptionLogic.ts @@ -4,7 +4,7 @@ import { loaders } from 'kea-loaders' import { beforeUnload, router, urlToAction } from 'kea-router' import api from 'lib/api' import { dayjs } from 'lib/dayjs' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { isEmail, isURL } from 'lib/utils' import { getInsightId } from 'scenes/insights/utils' import { integrationsLogic } from 'scenes/settings/project/integrationsLogic' diff --git a/frontend/src/lib/components/Support/supportLogic.ts b/frontend/src/lib/components/Support/supportLogic.ts index 98f24f8328e72..bac86a5a5c59d 100644 --- a/frontend/src/lib/components/Support/supportLogic.ts +++ b/frontend/src/lib/components/Support/supportLogic.ts @@ -4,7 +4,7 @@ import { actions, connect, kea, listeners, path, props, reducers, selectors } fr import { forms } from 'kea-forms' import { actionToUrl, router, urlToAction } from 'kea-router' import { FEATURE_FLAGS } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { uuid } from 'lib/utils' import posthog from 'posthog-js' @@ -62,11 +62,11 @@ export const TARGET_AREA_TO_NAME = { data_management: 'Data Management', data_warehouse: 'Data Warehouse', ingestion: 'Event Ingestion', - experiments: 'Experiments', + experiments: 'A/B Testing', feature_flags: 'Feature Flags', analytics: 'Product Analytics (Insights, Dashboards, Annotations)', session_replay: 'Session Replay (Recordings)', - toolbar: 'Toolbar & heatmaps', + toolbar: 'Toolbar & Heatmaps', surveys: 'Surveys', web_analytics: 'Web Analytics', 'posthog-3000': 'PostHog 3000', diff --git a/frontend/src/lib/components/Table/Table.tsx b/frontend/src/lib/components/Table/Table.tsx index 7cf69790d2763..afcecc0eb15c9 100644 --- a/frontend/src/lib/components/Table/Table.tsx +++ b/frontend/src/lib/components/Table/Table.tsx @@ -31,9 +31,7 @@ export function createdByColumn = Record - {item.created_by && ( - - )} + {item.created_by && } {/* eslint-disable-next-line react/forbid-dom-props */}
{item.created_by ? item.created_by.first_name || item.created_by.email : '-'} diff --git a/frontend/src/lib/components/TaxonomicPopover/TaxonomicPopover.tsx b/frontend/src/lib/components/TaxonomicPopover/TaxonomicPopover.tsx index 35eae8a066bbd..9220bf6477c4c 100644 --- a/frontend/src/lib/components/TaxonomicPopover/TaxonomicPopover.tsx +++ b/frontend/src/lib/components/TaxonomicPopover/TaxonomicPopover.tsx @@ -75,47 +75,45 @@ export function TaxonomicPopover - { - onChange?.(payload as ValueType, type, item) - setVisible(false) - }} - taxonomicGroupTypes={groupTypes ?? [groupType]} - eventNames={eventNames} - hogQLTable={hogQLTable} - excludedProperties={excludedProperties} - /> - } - sameWidth={false} - actionable - visible={visible} - onClickOutside={() => { - setVisible(false) - }} - > - {isClearButtonShown ? ( - , - tooltip: 'Clear selection', - onClick: (e) => { - e.stopPropagation() - onChange?.('' as ValueType, groupType, null) - setLocalValue('' as ValueType) - }, - divider: false, - }} - {...buttonPropsFinal} - /> - ) : ( - - )} - -
+ { + onChange?.(payload as ValueType, type, item) + setVisible(false) + }} + taxonomicGroupTypes={groupTypes ?? [groupType]} + eventNames={eventNames} + hogQLTable={hogQLTable} + excludedProperties={excludedProperties} + /> + } + sameWidth={false} + actionable + visible={visible} + onClickOutside={() => { + setVisible(false) + }} + > + {isClearButtonShown ? ( + , + tooltip: 'Clear selection', + onClick: (e) => { + e.stopPropagation() + onChange?.('' as ValueType, groupType, null) + setLocalValue('' as ValueType) + }, + divider: false, + }} + {...buttonPropsFinal} + /> + ) : ( + + )} + ) } diff --git a/frontend/src/lib/components/UserActivityIndicator/UserActivityIndicator.tsx b/frontend/src/lib/components/UserActivityIndicator/UserActivityIndicator.tsx index cd5f5cd1e2d70..8f8e602b6103f 100644 --- a/frontend/src/lib/components/UserActivityIndicator/UserActivityIndicator.tsx +++ b/frontend/src/lib/components/UserActivityIndicator/UserActivityIndicator.tsx @@ -27,7 +27,7 @@ export function UserActivityIndicator({ {at && } {by && by}
- {by && } + {by && }
) : null } diff --git a/frontend/src/lib/components/UserSelectItem.tsx b/frontend/src/lib/components/UserSelectItem.tsx index 0732418c35479..7990f4c8a7301 100644 --- a/frontend/src/lib/components/UserSelectItem.tsx +++ b/frontend/src/lib/components/UserSelectItem.tsx @@ -10,7 +10,7 @@ export interface UserSelectItemProps { export function UserSelectItem({ user }: UserSelectItemProps): JSX.Element { return ( - + {user.first_name} {`<${user.email}>`} diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index a6d1176087685..45b08ed3c9627 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -193,6 +193,7 @@ export const FEATURE_FLAGS = { SESSION_REPLAY_IOS: 'session-replay-ios', // owner: #team-replay YEAR_IN_HOG: 'year-in-hog', // owner: #team-replay SESSION_REPLAY_EXPORT_MOBILE_DATA: 'session-replay-export-mobile-data', // owner: #team-replay + REDIRECT_INGESTION_PRODUCT_ANALYTICS_ONBOARDING: 'redirect-ingestion-product-analytics-onboarding', // owner: @biancayang } as const export type FeatureFlagKey = (typeof FEATURE_FLAGS)[keyof typeof FEATURE_FLAGS] diff --git a/frontend/src/lib/lemon-ui/LemonButton/LemonButton.scss b/frontend/src/lib/lemon-ui/LemonButton/LemonButton.scss index 14d6014f2055b..95cf04c36bf94 100644 --- a/frontend/src/lib/lemon-ui/LemonButton/LemonButton.scss +++ b/frontend/src/lib/lemon-ui/LemonButton/LemonButton.scss @@ -164,6 +164,11 @@ .LemonButtonWithSideAction { position: relative; + width: fit-content; + + &--full-width { + width: 100%; + } } .LemonButtonWithSideAction__spacer { diff --git a/frontend/src/lib/lemon-ui/LemonButton/LemonButton.tsx b/frontend/src/lib/lemon-ui/LemonButton/LemonButton.tsx index f1be7aa2ff855..7cd59c8a401b4 100644 --- a/frontend/src/lib/lemon-ui/LemonButton/LemonButton.tsx +++ b/frontend/src/lib/lemon-ui/LemonButton/LemonButton.tsx @@ -246,7 +246,12 @@ export const LemonButton: React.FunctionComponent +
{workingButton}
{ + const isControlled = visible !== undefined + const [, parentPopoverLevel] = useContext(PopoverOverlayContext) const [localVisible, setLocalVisible] = useState(visible ?? false) @@ -46,9 +48,12 @@ export const LemonDropdown: React.FunctionComponent { - onVisibilityChange?.(localVisible) - }, [localVisible, onVisibilityChange]) + const setVisible = (value: boolean): void => { + if (!isControlled) { + setLocalVisible(value) + } + onVisibilityChange?.(value) + } return ( { if (trigger === 'click') { - setLocalVisible(false) + setVisible(false) } onClickOutside?.(e) }} onClickInside={(e) => { e.stopPropagation() - closeOnClickInside && setLocalVisible(false) + closeOnClickInside && setVisible(false) onClickInside?.(e) }} onMouseLeaveInside={(e) => { if (trigger === 'hover' && !referenceRef.current?.contains(e.relatedTarget as Node)) { - setLocalVisible(false) + setVisible(false) } onMouseLeaveInside?.(e) }} @@ -77,7 +82,7 @@ export const LemonDropdown: React.FunctionComponent {React.cloneElement(children, { onClick: (e: React.MouseEvent): void => { - setLocalVisible((state) => !state) + setVisible(!effectiveVisible) children.props.onClick?.(e) if (parentPopoverLevel > -1) { // If this button is inside another popover, let's not propagate this event so that @@ -87,12 +92,12 @@ export const LemonDropdown: React.FunctionComponent { if (trigger === 'hover') { - setLocalVisible(true) + setVisible(true) } }, onMouseLeave: (e: React.MouseEvent): void => { if (trigger === 'hover' && !floatingRef.current?.contains(e.relatedTarget as Node)) { - setLocalVisible(false) + setVisible(false) } }, 'aria-haspopup': 'true', diff --git a/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.scss b/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.scss deleted file mode 100644 index 03a754be1f978..0000000000000 --- a/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.scss +++ /dev/null @@ -1,31 +0,0 @@ -body:not(.posthog-3000) { - .LemonSelect--clearable { - padding-right: 0 !important; - - > span { - padding-right: 0 !important; - } - - .LemonButton__content { - gap: 0.5rem; - - .LemonSelect--button--clearable { - margin-left: auto; - } - } - } -} - -.posthog-3000 { - .LemonButton.LemonSelect--clearable { - .LemonButton__content { - gap: 0.25rem; - - .LemonSelect--button--clearable { - .LemonButton__chrome { - padding: 0.0625rem !important; - } - } - } - } -} diff --git a/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.stories.tsx b/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.stories.tsx index 6b78f42d26663..5fe4835da5abd 100644 --- a/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.stories.tsx +++ b/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.stories.tsx @@ -21,7 +21,7 @@ export default meta const Template: StoryFn = (props: LemonSelectProps) => { return (
- {(['small', undefined] as const).map((size, index) => ( + {(['small', 'medium', 'large', undefined] as const).map((size, index) => (
size={capitalizeFirstLetter(size || 'unspecified')}
diff --git a/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.tsx b/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.tsx index fda1df597097c..e5b596203e354 100644 --- a/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.tsx +++ b/frontend/src/lib/lemon-ui/LemonSelect/LemonSelect.tsx @@ -1,5 +1,3 @@ -import './LemonSelect.scss' - import clsx from 'clsx' import React, { useMemo } from 'react' @@ -143,12 +141,21 @@ export function LemonSelect({ closeParentPopoverOnClickInside={menu?.closeParentPopoverOnClickInside} > : undefined} type="secondary" status="stealth" + sideAction={ + isClearButtonShown + ? { + icon: , + divider: false, + onClick: () => { + onChange?.(null as T) + }, + } + : null + } {...buttonProps} > @@ -158,19 +165,6 @@ export function LemonSelect({ ? activeLeaf.label : value ?? {placeholder}} - {isClearButtonShown && ( - } - tooltip="Clear selection" - onClick={() => { - onChange?.(null as T) - }} - /> - )} ) diff --git a/frontend/src/lib/lemon-ui/LemonSelectMultiple/LemonSelectMultiple.stories.tsx b/frontend/src/lib/lemon-ui/LemonSelectMultiple/LemonSelectMultiple.stories.tsx index c360b9f86e796..baa2f805f48e4 100644 --- a/frontend/src/lib/lemon-ui/LemonSelectMultiple/LemonSelectMultiple.stories.tsx +++ b/frontend/src/lib/lemon-ui/LemonSelectMultiple/LemonSelectMultiple.stories.tsx @@ -16,7 +16,13 @@ const meta: Meta = { [`user-${i}`]: { labelComponent: ( - + {capitalizeFirstLetter(x)} {`<${x}@posthog.com>`} diff --git a/frontend/src/lib/lemon-ui/LemonSnack/LemonSnack.stories.tsx b/frontend/src/lib/lemon-ui/LemonSnack/LemonSnack.stories.tsx index e7e2c9528687d..cffe497f947bb 100644 --- a/frontend/src/lib/lemon-ui/LemonSnack/LemonSnack.stories.tsx +++ b/frontend/src/lib/lemon-ui/LemonSnack/LemonSnack.stories.tsx @@ -44,7 +44,7 @@ export const ComplexContent: Story = BasicTemplate.bind({}) ComplexContent.args = { children: ( - + Look at me I'm bold! diff --git a/frontend/src/lib/lemon-ui/LemonTable/LemonTable.scss b/frontend/src/lib/lemon-ui/LemonTable/LemonTable.scss index f21486605bd5f..1f6d8c6db99dd 100644 --- a/frontend/src/lib/lemon-ui/LemonTable/LemonTable.scss +++ b/frontend/src/lib/lemon-ui/LemonTable/LemonTable.scss @@ -12,7 +12,8 @@ border-radius: var(--radius); .WebAnalyticsDashboard &, - .Insight & { + .Insight &, + .InsightCard__viz & { // Special override for scenes where the table is primarily next to insights --lemon-table-background-color: var(--bg-light); } diff --git a/frontend/src/lib/lemon-ui/LemonTable/columnUtils.tsx b/frontend/src/lib/lemon-ui/LemonTable/columnUtils.tsx index 935dc5500b9ef..b258ba98de193 100644 --- a/frontend/src/lib/lemon-ui/LemonTable/columnUtils.tsx +++ b/frontend/src/lib/lemon-ui/LemonTable/columnUtils.tsx @@ -39,9 +39,7 @@ export function createdByColumn const { created_by } = item return (
- {created_by && ( - - )} + {created_by && }
) }, diff --git a/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.tsx b/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.tsx index dbc2181d76b05..1bbc2bbf7710b 100644 --- a/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.tsx +++ b/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.tsx @@ -6,7 +6,7 @@ import { TextContent } from 'lib/components/Cards/TextCard/TextCard' import { useUploadFiles } from 'lib/hooks/useUploadFiles' import { IconMarkdown, IconTools } from 'lib/lemon-ui/icons' import { LemonFileInput } from 'lib/lemon-ui/LemonFileInput/LemonFileInput' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { Link } from 'lib/lemon-ui/Link' import { Tooltip } from 'lib/lemon-ui/Tooltip' import posthog from 'posthog-js' diff --git a/frontend/src/lib/lemon-ui/LemonToast/LemonToast.stories.tsx b/frontend/src/lib/lemon-ui/LemonToast/LemonToast.stories.tsx new file mode 100644 index 0000000000000..d397becc87e43 --- /dev/null +++ b/frontend/src/lib/lemon-ui/LemonToast/LemonToast.stories.tsx @@ -0,0 +1,127 @@ +import { Meta, StoryObj } from '@storybook/react' +import { useEffect } from 'react' +import { Slide, ToastContainer } from 'react-toastify' + +import { lemonToast, ToastCloseButton, ToastContent, ToastContentProps } from './LemonToast' + +const meta: Meta = { + title: 'Lemon UI/Lemon Toast', + component: ToastContent, + parameters: { + testOptions: { + include3000: true, + waitForLoadersToDisappear: false, + snapshotTargetSelector: '.Toastify__toast-container', + }, + }, +} + +type ToastStory = { + toasts: ToastContentProps[] +} + +export default meta +type Story = StoryObj + +export const ToastTypes: Story = { + args: { + toasts: [ + { + type: 'info', + message: 'An info toast', + }, + { + type: 'success', + message: 'A success toast', + }, + { + type: 'warning', + message: 'A warning toast', + }, + { + type: 'error', + message: 'An error toast', + }, + ], + }, + render: (args, { globals }) => { + const isDarkModeOn = globals.theme === 'dark' + + useEffect(() => { + lemonToast.dismiss() + args.toasts.forEach((toast) => { + const { type, message, ...rest } = toast + lemonToast[type](message, rest) + }) + }, [isDarkModeOn]) + + return ( + } + theme={isDarkModeOn ? 'dark' : 'light'} + /> + ) + }, +} + +export const BillingError: Story = { + ...ToastTypes, + args: { + toasts: [ + { + type: 'error', + message: + 'Load experiment failed: This feature is part of the premium PostHog offering. To use it, subscribe to PostHog Cloud with a generous free tier: https://app.posthog.com/organization/billing', + }, + ], + }, +} + +export const WithButton: Story = { + ...ToastTypes, + args: { + toasts: [ + { + type: 'success', + message: 'Insight added to dashboard', + button: { + label: 'View dashboard', + action: (): void => {}, + }, + }, + ], + }, +} + +export const WithProgress: Story = { + ...ToastTypes, + args: { + toasts: [ + { + type: 'info', + message: 'An info toast with progress', + progress: 0.4, + } as ToastContentProps, + { + type: 'success', + message: 'A success toast with progress', + progress: 0.4, + } as ToastContentProps, + { + type: 'warning', + message: 'A warning toast with progress', + progress: 0.4, + } as ToastContentProps, + { + type: 'error', + message: 'An error toast with progress', + progress: 0.4, + } as ToastContentProps, + ], + }, +} diff --git a/frontend/src/lib/lemon-ui/lemonToast.tsx b/frontend/src/lib/lemon-ui/LemonToast/LemonToast.tsx similarity index 96% rename from frontend/src/lib/lemon-ui/lemonToast.tsx rename to frontend/src/lib/lemon-ui/LemonToast/LemonToast.tsx index 5af288acc6c96..03e57b9fc62d6 100644 --- a/frontend/src/lib/lemon-ui/lemonToast.tsx +++ b/frontend/src/lib/lemon-ui/LemonToast/LemonToast.tsx @@ -1,9 +1,9 @@ -import { IconCheckmark, IconClose, IconErrorOutline, IconInfo, IconWarning } from 'lib/lemon-ui/icons' -import { Spinner } from 'lib/lemon-ui/Spinner/Spinner' import posthog from 'posthog-js' import { toast, ToastContentProps as ToastifyRenderProps, ToastOptions } from 'react-toastify' -import { LemonButton } from './LemonButton' +import { IconCheckmark, IconClose, IconErrorOutline, IconInfo, IconWarning } from '../icons' +import { LemonButton } from '../LemonButton' +import { Spinner } from '../Spinner' export function ToastCloseButton({ closeToast }: { closeToast?: () => void }): JSX.Element { return ( @@ -44,7 +44,7 @@ export interface ToastContentProps { export function ToastContent({ type, message, button, id }: ToastContentProps): JSX.Element { return (
- {message} + {message} {button && ( { diff --git a/frontend/src/lib/lemon-ui/LemonToast/index.ts b/frontend/src/lib/lemon-ui/LemonToast/index.ts new file mode 100644 index 0000000000000..f8ca437c849ab --- /dev/null +++ b/frontend/src/lib/lemon-ui/LemonToast/index.ts @@ -0,0 +1 @@ +export { lemonToast } from './LemonToast' diff --git a/frontend/src/lib/lemon-ui/Link/Link.tsx b/frontend/src/lib/lemon-ui/Link/Link.tsx index 3f7813e75f070..0289c79e8b69f 100644 --- a/frontend/src/lib/lemon-ui/Link/Link.tsx +++ b/frontend/src/lib/lemon-ui/Link/Link.tsx @@ -1,15 +1,11 @@ import './Link.scss' import clsx from 'clsx' -import { useActions } from 'kea' import { router } from 'kea-router' -import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' import { isExternalLink } from 'lib/utils' import React from 'react' import { useNotebookDrag } from 'scenes/notebooks/AddToNotebook/DraggableToNotebook' -import { sidePanelDocsLogic } from '~/layout/navigation-3000/sidepanel/panels/sidePanelDocsLogic' - import { IconOpenInNew } from '../icons' import { Tooltip } from '../Tooltip' @@ -54,9 +50,10 @@ const isPostHogDomain = (url: string): boolean => { return /^https:\/\/((www|app|eu)\.)?posthog\.com/.test(url) } -const isPostHogComDocs = (url: string): boolean => { - return /^https:\/\/(www\.)?posthog\.com\/docs/.test(url) -} +// NOTE: Temporarily disabled - owner @corywatilo +// const isPostHogComDocs = (url: string): boolean => { +// return /^https:\/\/(www\.)?posthog\.com\/docs/.test(url) +// } /** * Link @@ -87,8 +84,9 @@ export const Link: React.FC> = Reac href: typeof to === 'string' ? to : undefined, }) - const is3000 = useFeatureFlag('POSTHOG_3000', 'test') - const { openDocsPage } = useActions(sidePanelDocsLogic) + // NOTE: Temporarily disabled - owner @corywatilo + // const is3000 = useFeatureFlag('POSTHOG_3000', 'test') + // const { openSidePanel } = useActions(sidePanelStateLogic) const onClick = (event: React.MouseEvent): void => { if (event.metaKey || event.ctrlKey) { @@ -103,11 +101,12 @@ export const Link: React.FC> = Reac return } - if (typeof to === 'string' && is3000 && isPostHogComDocs(to)) { - event.preventDefault() - openDocsPage(to) - return - } + // NOTE: Temporarily disabled - owner @corywatilo + // if (typeof to === 'string' && is3000 && isPostHogComDocs(to)) { + // event.preventDefault() + // openSidePanel(SidePanelTab.Docs, to) + // return + // } if (!target && to && !isExternalLink(to) && !disableClientSideRouting && !shouldForcePageLoad(to)) { event.preventDefault() diff --git a/frontend/src/lib/lemon-ui/Popover/Popover.tsx b/frontend/src/lib/lemon-ui/Popover/Popover.tsx index 0541bbade9e36..fe42c89c98fa9 100644 --- a/frontend/src/lib/lemon-ui/Popover/Popover.tsx +++ b/frontend/src/lib/lemon-ui/Popover/Popover.tsx @@ -138,7 +138,11 @@ export const Popover = React.forwardRef(function P }) const [floatingElement, setFloatingElement] = useState(null) - const mergedReferenceRef = useMergeRefs([referenceRef, extraReferenceRef || null]) as React.RefCallback + const mergedReferenceRef = useMergeRefs([ + referenceRef, + extraReferenceRef || null, + (children as any)?.ref, + ]) as React.RefCallback const arrowStyle = middlewareData.arrow ? { diff --git a/frontend/src/lib/lemon-ui/ProfilePicture/ProfileBubbles.tsx b/frontend/src/lib/lemon-ui/ProfilePicture/ProfileBubbles.tsx index 23cd6b167efeb..0da16a220f548 100644 --- a/frontend/src/lib/lemon-ui/ProfilePicture/ProfileBubbles.tsx +++ b/frontend/src/lib/lemon-ui/ProfilePicture/ProfileBubbles.tsx @@ -29,8 +29,10 @@ export function ProfileBubbles({ people, tooltip, limit = 6, ...divProps }: Prof {shownPeople.map(({ email, name, title }, index) => ( , 'first_name' | 'email' | 'last_name'> | null name?: string - email?: string size?: 'xs' | 'sm' | 'md' | 'lg' | 'xl' | 'xxl' showName?: boolean className?: string @@ -22,8 +24,8 @@ export interface ProfilePictureProps { } export function ProfilePicture({ + user, name, - email, size = 'lg', showName, className, @@ -31,9 +33,16 @@ export function ProfilePicture({ title, type = 'person', }: ProfilePictureProps): JSX.Element { - const { user } = useValues(userLogic) + const { user: currentUser } = useValues(userLogic) const [gravatarLoaded, setGravatarLoaded] = useState() + let email = user?.email + + if (user) { + name = fullName(user) + email = user.email + } + const combinedNameAndEmail = name && email ? `${name} <${email}>` : name || email const gravatarUrl = useMemo(() => { @@ -82,7 +91,9 @@ export function ProfilePicture({ ) : (
{pictureComponent} - {user?.email === email ? 'you' : name || email || 'an unknown user'} + + {currentUser?.email === email ? 'you' : name || email || 'an unknown user'} +
) } diff --git a/frontend/src/lib/lemon-ui/icons/icons.tsx b/frontend/src/lib/lemon-ui/icons/icons.tsx index e6f8a3d8a4fb4..45c3de18cfa09 100644 --- a/frontend/src/lib/lemon-ui/icons/icons.tsx +++ b/frontend/src/lib/lemon-ui/icons/icons.tsx @@ -9,6 +9,7 @@ interface IconWithCountProps { count: number showZero?: boolean status?: LemonBadgeProps['status'] + className?: string } export function IconWithCount({ @@ -16,9 +17,10 @@ export function IconWithCount({ children, showZero, status = 'primary', + className, }: PropsWithChildren): JSX.Element { return ( - + {children} diff --git a/frontend/src/lib/utils.tsx b/frontend/src/lib/utils.tsx index 4477d4d3ab553..7e73d58fc2d0f 100644 --- a/frontend/src/lib/utils.tsx +++ b/frontend/src/lib/utils.tsx @@ -191,6 +191,10 @@ export function capitalizeFirstLetter(string: string): string { return string.charAt(0).toUpperCase() + string.slice(1) } +export function fullName(props: { first_name?: string; last_name?: string }): string { + return `${props.first_name || ''} ${props.last_name || ''}`.trim() +} + export const genericOperatorMap: Record = { exact: '= equals', is_not: "≠ doesn't equal", diff --git a/frontend/src/loadPostHogJS.tsx b/frontend/src/loadPostHogJS.tsx index 807fce2883849..d56a7545adde3 100644 --- a/frontend/src/loadPostHogJS.tsx +++ b/frontend/src/loadPostHogJS.tsx @@ -37,6 +37,7 @@ export function loadPostHogJS(): void { posthog.opt_in_capturing() } }, + __preview_measure_pageview_stats: true, }) ) diff --git a/frontend/src/models/dashboardsModel.tsx b/frontend/src/models/dashboardsModel.tsx index 54d52b20ad986..7ba4c285067e9 100644 --- a/frontend/src/models/dashboardsModel.tsx +++ b/frontend/src/models/dashboardsModel.tsx @@ -3,7 +3,7 @@ import { loaders } from 'kea-loaders' import { router, urlToAction } from 'kea-router' import api, { PaginatedResponse } from 'lib/api' import { GENERATED_DASHBOARD_PREFIX } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { idToKey, isUserLoggedIn } from 'lib/utils' import { DashboardEventSource, eventUsageLogic } from 'lib/utils/eventUsageLogic' import { permanentlyMount } from 'lib/utils/kea-logic-builders' diff --git a/frontend/src/models/insightsModel.tsx b/frontend/src/models/insightsModel.tsx index 02e0e5f21d588..71bca8a4ae8b7 100644 --- a/frontend/src/models/insightsModel.tsx +++ b/frontend/src/models/insightsModel.tsx @@ -1,6 +1,6 @@ import { actions, connect, kea, listeners, path } from 'kea' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { promptLogic } from 'lib/logic/promptLogic' import { teamLogic } from 'scenes/teamLogic' diff --git a/frontend/src/queries/QueryEditor/queryEditorLogic.ts b/frontend/src/queries/QueryEditor/queryEditorLogic.ts index 585f1b84db761..2abeb3cd3685a 100644 --- a/frontend/src/queries/QueryEditor/queryEditorLogic.ts +++ b/frontend/src/queries/QueryEditor/queryEditorLogic.ts @@ -1,5 +1,5 @@ import { actions, kea, key, listeners, path, props, propsChanged, reducers, selectors } from 'kea' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { QueryEditorProps } from '~/queries/QueryEditor/QueryEditor' import { Node } from '~/queries/schema' diff --git a/frontend/src/queries/nodes/DataTable/DataTable.tsx b/frontend/src/queries/nodes/DataTable/DataTable.tsx index 114d8811494f1..7cfec3e91a888 100644 --- a/frontend/src/queries/nodes/DataTable/DataTable.tsx +++ b/frontend/src/queries/nodes/DataTable/DataTable.tsx @@ -474,26 +474,7 @@ export function DataTable({ uniqueKey, query, setQuery, context, cachedResults } ) /* Bust the LemonTable cache when columns change */ } dataSource={dataTableRows ?? []} - rowKey={({ result }: DataTableRow, rowIndex) => { - if (result) { - if ( - sourceFeatures.has(QueryFeature.resultIsArrayOfArrays) && - sourceFeatures.has(QueryFeature.columnsInResponse) - ) { - if (columnsInResponse?.includes('*')) { - return result[columnsInResponse.indexOf('*')].uuid - } else if (columnsInResponse?.includes('uuid')) { - return result[columnsInResponse.indexOf('uuid')] - } else if (columnsInResponse?.includes('id')) { - return result[columnsInResponse.indexOf('id')] - } - } - return ( - (result && 'uuid' in result ? (result as any).uuid : null) ?? - (result && 'id' in result ? (result as any).id : null) ?? - JSON.stringify(result ?? rowIndex) - ) - } + rowKey={(_, rowIndex) => { return rowIndex }} sorting={null} diff --git a/frontend/src/queries/nodes/DataTable/DataTableExport.tsx b/frontend/src/queries/nodes/DataTable/DataTableExport.tsx index a1b04d60e6579..8b1e0f4a5ad13 100644 --- a/frontend/src/queries/nodes/DataTable/DataTableExport.tsx +++ b/frontend/src/queries/nodes/DataTable/DataTableExport.tsx @@ -44,7 +44,7 @@ async function startDownload(query: DataTableNode, onlySelectedColumns: boolean) ) } else if (isPersonsNode(query.source)) { exportContext['columns'] = exportContext['columns'].map((c: string) => - removeExpressionComment(c) === 'person' ? 'properties.email' : c + removeExpressionComment(c) === 'person' ? 'email' : c ) } } diff --git a/frontend/src/queries/nodes/DataVisualization/Components/Charts/LineGraph.tsx b/frontend/src/queries/nodes/DataVisualization/Components/Charts/LineGraph.tsx index 0d2e76f3d8ff1..3869c45c89615 100644 --- a/frontend/src/queries/nodes/DataVisualization/Components/Charts/LineGraph.tsx +++ b/frontend/src/queries/nodes/DataVisualization/Components/Charts/LineGraph.tsx @@ -166,7 +166,6 @@ export const LineGraph = (): JSX.Element => { tooltipEl.classList.remove('above', 'below', 'no-transform') tooltipEl.classList.add(tooltip.yAlign || 'no-transform') tooltipEl.style.opacity = '1' - tooltipEl.style.display = 'initial' if (tooltip.body) { const referenceDataPoint = tooltip.dataPoints[0] // Use this point as reference to get the date @@ -225,8 +224,8 @@ export const LineGraph = (): JSX.Element => { ? chartClientLeft + tooltip.caretX - tooltipEl.clientWidth - 8 // If tooltip is too large (or close to the edge), show it to the left of the data point instead : defaultOffsetLeft - tooltipEl.style.top = Math.min(tooltipClientTop, window.innerHeight) + 'px' - tooltipEl.style.left = Math.min(tooltipClientLeft, window.innerWidth) + 'px' + tooltipEl.style.top = tooltipClientTop + 'px' + tooltipEl.style.left = tooltipClientLeft + 'px' }, }, }, diff --git a/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts b/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts index f64615df762e2..39e1da3ebc216 100644 --- a/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts +++ b/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts @@ -85,7 +85,7 @@ export const hogQLQueryEditorLogic = kea([ ], aiAvailable: [() => [preflightLogic.selectors.preflight], (preflight) => preflight?.openai_available], }), - listeners(({ actions, asyncActions, props, values }) => ({ + listeners(({ actions, props, values }) => ({ saveQuery: () => { const query = values.queryInput // TODO: Is below line necessary if the only way for queryInput to change is already through setQueryInput? @@ -181,7 +181,7 @@ export const hogQLQueryEditorLogic = kea([ kind: NodeKind.HogQLQuery, query: values.queryInput, } - await asyncActions.createDataWarehouseSavedQuery({ name, query }) + await dataWarehouseSavedQueriesLogic.asyncActions.createDataWarehouseSavedQuery({ name, query }) }, })), ]) diff --git a/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts b/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts index 7b9da7b5fa98e..31628318c2e7c 100644 --- a/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts +++ b/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts @@ -234,7 +234,10 @@ export const filtersToQueryNode = (filters: Partial): InsightQueryNo breakdowns: filters.breakdowns, breakdown_group_type_index: filters.breakdown_group_type_index, ...(isTrendsFilter(filters) - ? { breakdown_histogram_bin_count: filters.breakdown_histogram_bin_count } + ? { + breakdown_histogram_bin_count: filters.breakdown_histogram_bin_count, + breakdown_hide_other_aggregation: filters.breakdown_hide_other_aggregation, + } : {}), }) } diff --git a/frontend/src/queries/nodes/InsightViz/InsightDisplayConfig.tsx b/frontend/src/queries/nodes/InsightViz/InsightDisplayConfig.tsx index b98757af47523..9a408b4e4fb39 100644 --- a/frontend/src/queries/nodes/InsightViz/InsightDisplayConfig.tsx +++ b/frontend/src/queries/nodes/InsightViz/InsightDisplayConfig.tsx @@ -159,7 +159,7 @@ export function InsightDisplayConfig(): JSX.Element { {advancedOptions.length > 0 && ( - + Options{advancedOptionsCount ? ` (${advancedOptionsCount})` : null} diff --git a/frontend/src/queries/nodes/PersonsNode/PersonsSearch.tsx b/frontend/src/queries/nodes/PersonsNode/PersonsSearch.tsx index b660556ea23a8..8836b76cac7de 100644 --- a/frontend/src/queries/nodes/PersonsNode/PersonsSearch.tsx +++ b/frontend/src/queries/nodes/PersonsNode/PersonsSearch.tsx @@ -4,12 +4,32 @@ import { Tooltip } from 'lib/lemon-ui/Tooltip' import { useDebouncedQuery } from '~/queries/hooks/useDebouncedQuery' import { PersonsNode, PersonsQuery } from '~/queries/schema' +import { isQueryForGroup } from '~/queries/utils' +type ActorType = 'person' | 'group' interface PersonSearchProps { query: PersonsNode | PersonsQuery setQuery?: (query: PersonsNode | PersonsQuery) => void } +interface LabelType { + label: string + description: string +} + +const labels: Record = { + person: { + label: 'persons', + description: + 'Search by email or Distinct ID. Email will match partially, for example: "@gmail.com". Distinct ID needs to match exactly.', + }, + group: { + label: 'groups', + description: + 'Search by group name or Distinct ID. Group name will match partially. Distinct ID needs to match exactly.', + }, +} + export function PersonsSearch({ query, setQuery }: PersonSearchProps): JSX.Element { const { value, onChange } = useDebouncedQuery( query, @@ -17,25 +37,19 @@ export function PersonsSearch({ query, setQuery }: PersonSearchProps): JSX.Eleme (query) => query.search || '', (query, value) => ({ ...query, search: value }) ) + const target: ActorType = isQueryForGroup(query) ? 'group' : 'person' return (
- - Search by email or Distinct ID. Email will match partially, for example: "@gmail.com". Distinct - ID needs to match exactly. - - } - > + {labels[target].description}}>
diff --git a/frontend/src/queries/nodes/TimeToSeeData/Trace/traceLogic.tsx b/frontend/src/queries/nodes/TimeToSeeData/Trace/traceLogic.tsx index c36416b2fe033..232748290d242 100644 --- a/frontend/src/queries/nodes/TimeToSeeData/Trace/traceLogic.tsx +++ b/frontend/src/queries/nodes/TimeToSeeData/Trace/traceLogic.tsx @@ -21,7 +21,7 @@ export function sessionNodeFacts(node: TimeToSeeNode): Record, + user: , duration: humanFriendlyMilliseconds(node.data.duration_ms) || 'unknown', sessionEventCount: node.data.events_count, frustratingInteractions: node.data.frustrating_interactions_count, diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index b441a7f004f3b..529fdc752f618 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -233,6 +233,9 @@ "breakdown_group_type_index": { "type": ["number", "null"] }, + "breakdown_hide_other_aggregation": { + "type": ["boolean", "null"] + }, "breakdown_histogram_bin_count": { "type": "number" }, @@ -1044,6 +1047,12 @@ "hogql": { "type": "string" }, + "limit": { + "type": "integer" + }, + "offset": { + "type": "integer" + }, "results": { "items": { "items": {}, @@ -1064,7 +1073,7 @@ "type": "array" } }, - "required": ["columns", "types", "results", "hogql"], + "required": ["columns", "types", "results", "hogql", "limit", "offset"], "type": "object" }, "FeaturePropertyFilter": { @@ -1113,6 +1122,9 @@ "breakdown_group_type_index": { "type": ["number", "null"] }, + "breakdown_hide_other_aggregation": { + "type": ["boolean", "null"] + }, "breakdown_normalize_url": { "type": "boolean" }, @@ -1304,7 +1316,7 @@ "properties": { "aggregation_group_type_index": { "description": "Groups aggregation", - "type": "number" + "type": "integer" }, "breakdown": { "$ref": "#/definitions/BreakdownFilter", @@ -1710,6 +1722,10 @@ "day": { "type": "string" }, + "interval": { + "description": "An interval selected out of available intervals in source query", + "type": "integer" + }, "kind": { "const": "InsightPersonsQuery", "type": "string" @@ -1814,7 +1830,7 @@ "properties": { "aggregation_group_type_index": { "description": "Groups aggregation", - "type": "number" + "type": "integer" }, "dateRange": { "$ref": "#/definitions/DateRange", @@ -2104,7 +2120,7 @@ "properties": { "aggregation_group_type_index": { "description": "Groups aggregation", - "type": "number" + "type": "integer" }, "dateRange": { "$ref": "#/definitions/DateRange", @@ -2227,10 +2243,10 @@ "type": "string" }, "limit": { - "type": "number" + "type": "integer" }, "offset": { - "type": "number" + "type": "integer" }, "orderBy": { "items": { @@ -2284,6 +2300,15 @@ "hogql": { "type": "string" }, + "limit": { + "type": "integer" + }, + "missing_actors_count": { + "type": "integer" + }, + "offset": { + "type": "integer" + }, "results": { "items": { "items": {}, @@ -2304,7 +2329,7 @@ "type": "array" } }, - "required": ["results", "columns", "types", "hogql"], + "required": ["results", "columns", "types", "hogql", "limit", "offset"], "type": "object" }, "PropertyFilterType": { @@ -2628,7 +2653,7 @@ "properties": { "aggregation_group_type_index": { "description": "Groups aggregation", - "type": "number" + "type": "integer" }, "dateRange": { "$ref": "#/definitions/DateRange", @@ -3206,7 +3231,7 @@ "properties": { "aggregation_group_type_index": { "description": "Groups aggregation", - "type": "number" + "type": "integer" }, "breakdown": { "$ref": "#/definitions/BreakdownFilter", @@ -3471,6 +3496,9 @@ "dateRange": { "$ref": "#/definitions/DateRange" }, + "includeScrollDepth": { + "type": "boolean" + }, "kind": { "const": "WebStatsTableQuery", "type": "string" diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index ec07dad2e1c27..94b739a189871 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -259,6 +259,10 @@ export interface EventsQueryResponse { hogql: string hasMore?: boolean timings?: QueryTiming[] + /** @asType integer */ + limit: number + /** @asType integer */ + offset: number } export interface EventsQueryPersonColumn { uuid: string @@ -464,7 +468,10 @@ export interface InsightsQueryBase extends Node { filterTestAccounts?: boolean /** Property filters for all series */ properties?: AnyPropertyFilter[] | PropertyGroupFilter - /** Groups aggregation */ + /** + * Groups aggregation + * @asType integer + **/ aggregation_group_type_index?: number /** Sampling rate */ samplingFactor?: number | null @@ -631,6 +638,12 @@ export interface PersonsQueryResponse { hogql: string timings?: QueryTiming[] hasMore?: boolean + /** @asType integer */ + limit: number + /** @asType integer */ + offset: number + /** @asType integer */ + missing_actors_count?: number } export interface PersonsQuery extends DataNode { @@ -641,7 +654,9 @@ export interface PersonsQuery extends DataNode { properties?: AnyPropertyFilter[] fixedProperties?: AnyPropertyFilter[] orderBy?: string[] + /** @asType integer */ limit?: number + /** @asType integer */ offset?: number response?: PersonsQueryResponse } @@ -729,6 +744,7 @@ export interface WebStatsTableQuery extends WebAnalyticsQueryBase { kind: NodeKind.WebStatsTableQuery breakdownBy: WebStatsBreakdown response?: WebStatsTableQueryResponse + includeScrollDepth?: boolean } export interface WebStatsTableQueryResponse extends QueryResponse { results: unknown[] @@ -765,6 +781,11 @@ export interface InsightPersonsQuery { source: InsightQueryNode day?: string status?: string + /** + * An interval selected out of available intervals in source query + * @asType integer + */ + interval?: number // TODO: add breakdowns // TODO: add fields for other insights (funnels dropdown, compare_previous choice, etc) response?: PersonsQueryResponse @@ -851,6 +872,7 @@ export interface BreakdownFilter { breakdowns?: Breakdown[] breakdown_group_type_index?: number | null breakdown_histogram_bin_count?: number // trends breakdown histogram bin count + breakdown_hide_other_aggregation?: boolean | null // hides the "other" field for trends } export interface DashboardFilter { diff --git a/frontend/src/queries/utils.ts b/frontend/src/queries/utils.ts index 86e1697b4be95..11dd7a60db071 100644 --- a/frontend/src/queries/utils.ts +++ b/frontend/src/queries/utils.ts @@ -183,6 +183,15 @@ export function isDatabaseSchemaQuery(node?: Node): node is DatabaseSchemaQuery return node?.kind === NodeKind.DatabaseSchemaQuery } +export function isQueryForGroup(query: PersonsNode | PersonsQuery): boolean { + return ( + isPersonsQuery(query) && + isInsightPersonsQuery(query.source) && + isRetentionQuery(query.source.source) && + query.source.source.aggregation_group_type_index !== undefined + ) +} + export function isInsightQueryWithSeries( node?: Node ): node is TrendsQuery | FunnelsQuery | StickinessQuery | LifecycleQuery { diff --git a/frontend/src/scenes/App.tsx b/frontend/src/scenes/App.tsx index e403138c07f86..d6a1994f9d107 100644 --- a/frontend/src/scenes/App.tsx +++ b/frontend/src/scenes/App.tsx @@ -1,7 +1,7 @@ import { actions, BindLogic, connect, events, kea, path, reducers, selectors, useMountedLogic, useValues } from 'kea' import { FEATURE_FLAGS } from 'lib/constants' import { use3000Body } from 'lib/hooks/use3000Body' -import { ToastCloseButton } from 'lib/lemon-ui/lemonToast' +import { ToastCloseButton } from 'lib/lemon-ui/LemonToast/LemonToast' import { SpinnerOverlay } from 'lib/lemon-ui/Spinner/Spinner' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { inAppPromptLogic } from 'lib/logic/inAppPrompt/inAppPromptLogic' diff --git a/frontend/src/scenes/actions/actionEditLogic.tsx b/frontend/src/scenes/actions/actionEditLogic.tsx index 5d3c8d193285f..bbd36a29a8404 100644 --- a/frontend/src/scenes/actions/actionEditLogic.tsx +++ b/frontend/src/scenes/actions/actionEditLogic.tsx @@ -3,7 +3,7 @@ import { forms } from 'kea-forms' import { loaders } from 'kea-loaders' import { beforeUnload, router, urlToAction } from 'kea-router' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { Link } from 'lib/lemon-ui/Link' import { uuid } from 'lib/utils' import { deleteWithUndo } from 'lib/utils/deleteWithUndo' diff --git a/frontend/src/scenes/annotations/Annotations.tsx b/frontend/src/scenes/annotations/Annotations.tsx index 90762f8c3bc81..110d8794fd0b1 100644 --- a/frontend/src/scenes/annotations/Annotations.tsx +++ b/frontend/src/scenes/annotations/Annotations.tsx @@ -102,8 +102,7 @@ export function Annotations(): JSX.Element { return (
- +
{user.first_name}
{user.organization?.name}
diff --git a/frontend/src/scenes/authentication/passwordResetLogic.ts b/frontend/src/scenes/authentication/passwordResetLogic.ts index 487510514e474..a0240f016fa36 100644 --- a/frontend/src/scenes/authentication/passwordResetLogic.ts +++ b/frontend/src/scenes/authentication/passwordResetLogic.ts @@ -3,7 +3,7 @@ import { forms } from 'kea-forms' import { loaders } from 'kea-loaders' import { urlToAction } from 'kea-router' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import type { passwordResetLogicType } from './passwordResetLogicType' diff --git a/frontend/src/scenes/authentication/signup/verify-email/verifyEmailLogic.ts b/frontend/src/scenes/authentication/signup/verify-email/verifyEmailLogic.ts index 8b28386fff3e3..55ef8dc85375d 100644 --- a/frontend/src/scenes/authentication/signup/verify-email/verifyEmailLogic.ts +++ b/frontend/src/scenes/authentication/signup/verify-email/verifyEmailLogic.ts @@ -2,7 +2,7 @@ import { actions, kea, path, reducers } from 'kea' import { loaders } from 'kea-loaders' import { urlToAction } from 'kea-router' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import type { verifyEmailLogicType } from './verifyEmailLogicType' diff --git a/frontend/src/scenes/cohorts/cohortEditLogic.ts b/frontend/src/scenes/cohorts/cohortEditLogic.ts index 15794de4b6d24..ecfba636b35c8 100644 --- a/frontend/src/scenes/cohorts/cohortEditLogic.ts +++ b/frontend/src/scenes/cohorts/cohortEditLogic.ts @@ -4,7 +4,7 @@ import { loaders } from 'kea-loaders' import { actionToUrl, router } from 'kea-router' import api from 'lib/api' import { ENTITY_MATCH_TYPE, FEATURE_FLAGS } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { NEW_COHORT, NEW_CRITERIA, NEW_CRITERIA_GROUP } from 'scenes/cohorts/CohortFilters/constants' import { diff --git a/frontend/src/scenes/dashboard/DashboardCollaborators.tsx b/frontend/src/scenes/dashboard/DashboardCollaborators.tsx index 3f43a36e43d42..bb61ce0f55110 100644 --- a/frontend/src/scenes/dashboard/DashboardCollaborators.tsx +++ b/frontend/src/scenes/dashboard/DashboardCollaborators.tsx @@ -121,7 +121,7 @@ function CollaboratorRow({ return (
- + = [ { @@ -246,20 +245,11 @@ export function DashboardsTable({
Created by: - ({ - value: x.user.uuid, - label: x.user.first_name, - })), - ]} + { - setFilters({ createdBy: v }) - }} - dropdownMatchSelectWidth={false} + type="secondary" + value={filters.createdBy === 'All users' ? null : filters.createdBy} + onChange={(user) => setFilters({ createdBy: user?.uuid || 'All users' })} />
{extraActions} diff --git a/frontend/src/scenes/dashboard/newDashboardLogic.ts b/frontend/src/scenes/dashboard/newDashboardLogic.ts index 455bfa884cdd7..eb7e20627e22f 100644 --- a/frontend/src/scenes/dashboard/newDashboardLogic.ts +++ b/frontend/src/scenes/dashboard/newDashboardLogic.ts @@ -3,7 +3,7 @@ import { forms } from 'kea-forms' import { router } from 'kea-router' import api from 'lib/api' import { DashboardRestrictionLevel } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' diff --git a/frontend/src/scenes/data-management/definition/definitionEditLogic.ts b/frontend/src/scenes/data-management/definition/definitionEditLogic.ts index 8b14c0f51d2a9..7d8b2fcdd456d 100644 --- a/frontend/src/scenes/data-management/definition/definitionEditLogic.ts +++ b/frontend/src/scenes/data-management/definition/definitionEditLogic.ts @@ -2,7 +2,7 @@ import { beforeUnmount, connect, kea, key, path, props } from 'kea' import { forms } from 'kea-forms' import { loaders } from 'kea-loaders' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { capitalizeFirstLetter } from 'lib/utils' import { definitionLogic, diff --git a/frontend/src/scenes/data-warehouse/viewLinkLogic.tsx b/frontend/src/scenes/data-warehouse/viewLinkLogic.tsx index c8f4bafc4999d..d51adf1b48123 100644 --- a/frontend/src/scenes/data-warehouse/viewLinkLogic.tsx +++ b/frontend/src/scenes/data-warehouse/viewLinkLogic.tsx @@ -2,7 +2,7 @@ import { actions, afterMount, connect, kea, listeners, path, reducers, selectors import { forms } from 'kea-forms' import { loaders } from 'kea-loaders' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic' import { DataWarehouseViewLink } from '~/types' diff --git a/frontend/src/scenes/events/createActionFromEvent.tsx b/frontend/src/scenes/events/createActionFromEvent.tsx index d12f365efb3a7..a0368eb41fef8 100644 --- a/frontend/src/scenes/events/createActionFromEvent.tsx +++ b/frontend/src/scenes/events/createActionFromEvent.tsx @@ -1,7 +1,7 @@ import { router } from 'kea-router' import { CLICK_TARGETS, elementToSelector, matchesDataAttribute } from 'lib/actionUtils' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { Link } from 'lib/lemon-ui/Link' import { autoCaptureEventToDescription } from 'lib/utils' import { urls } from 'scenes/urls' diff --git a/frontend/src/scenes/events/eventsSceneLogic.tsx b/frontend/src/scenes/events/eventsSceneLogic.tsx index 08912e5666d76..8a9b53bfb30ce 100644 --- a/frontend/src/scenes/events/eventsSceneLogic.tsx +++ b/frontend/src/scenes/events/eventsSceneLogic.tsx @@ -1,7 +1,7 @@ import equal from 'fast-deep-equal' import { actions, connect, kea, path, reducers, selectors } from 'kea' import { actionToUrl, urlToAction } from 'kea-router' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { objectsEqual } from 'lib/utils' import { getDefaultEventsSceneQuery } from 'scenes/events/defaults' import { teamLogic } from 'scenes/teamLogic' diff --git a/frontend/src/scenes/experiments/Experiment.scss b/frontend/src/scenes/experiments/Experiment.scss index c9c4fe75b35cf..633259b80037c 100644 --- a/frontend/src/scenes/experiments/Experiment.scss +++ b/frontend/src/scenes/experiments/Experiment.scss @@ -107,7 +107,6 @@ .exp-description { font-style: italic; - color: var(--muted); } .participants { @@ -209,3 +208,10 @@ border: 1px solid var(--border); border-radius: var(--radius); } + +.exp-flag-copy-label { + font-size: 11px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; +} diff --git a/frontend/src/scenes/experiments/Experiment.tsx b/frontend/src/scenes/experiments/Experiment.tsx index d5e034407cd37..8fda5a2f5559f 100644 --- a/frontend/src/scenes/experiments/Experiment.tsx +++ b/frontend/src/scenes/experiments/Experiment.tsx @@ -625,7 +625,21 @@ export function Experiment(): JSX.Element { } />
- +
+ {experiment.feature_flag && ( + <> +
Feature flag
+ + {experiment.feature_flag.key} + + + )} +
+
{isExperimentRunning ? ( {experiment.description || 'There is no description for this experiment.'} )} - +
diff --git a/frontend/src/scenes/experiments/Experiments.tsx b/frontend/src/scenes/experiments/Experiments.tsx index 5f0229f634a50..70ba21a643296 100644 --- a/frontend/src/scenes/experiments/Experiments.tsx +++ b/frontend/src/scenes/experiments/Experiments.tsx @@ -2,6 +2,7 @@ import { LemonInput, LemonSelect } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { router } from 'kea-router' import { ExperimentsHog } from 'lib/components/hedgehogs' +import { MemberSelect } from 'lib/components/MemberSelect' import { PageHeader } from 'lib/components/PageHeader' import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction' import { normalizeColumnTitle } from 'lib/components/Table/utils' @@ -38,8 +39,10 @@ export function Experiments(): JSX.Element { searchTerm, shouldShowEmptyState, shouldShowProductIntroduction, + searchStatus, + userFilter, } = useValues(experimentsLogic) - const { setExperimentsTab, deleteExperiment, archiveExperiment, setSearchStatus, setSearchTerm } = + const { setExperimentsTab, deleteExperiment, archiveExperiment, setSearchStatus, setSearchTerm, setUserFilter } = useActions(experimentsLogic) const { hasAvailableFeature } = useValues(userLogic) @@ -229,6 +232,7 @@ export function Experiments(): JSX.Element { Status { if (status) { setSearchStatus(status as ProgressStatus | 'all') @@ -242,9 +246,20 @@ export function Experiments(): JSX.Element { { label: 'Complete', value: ProgressStatus.Complete }, ] as { label: string; value: string }[] } - value="all" + value={searchStatus ?? 'all'} + dropdownMatchSelectWidth={false} dropdownMaxContentWidth /> + + Created by + + setUserFilter(user?.uuid ?? null)} + />
([ setSearchTerm: (searchTerm: string) => ({ searchTerm }), setSearchStatus: (status: ProgressStatus | 'all') => ({ status }), setExperimentsTab: (tabKey: ExperimentsTabs) => ({ tabKey }), + setUserFilter: (userFilter: string | null) => ({ userFilter }), }), reducers({ searchTerm: { @@ -58,6 +59,12 @@ export const experimentsLogic = kea([ searchStatus: { setSearchStatus: (_, { status }) => status, }, + userFilter: [ + null as string | null, + { + setUserFilter: (_, { userFilter }) => userFilter, + }, + ], tab: [ ExperimentsTabs.All as ExperimentsTabs, { @@ -97,8 +104,8 @@ export const experimentsLogic = kea([ })), selectors(({ values }) => ({ filteredExperiments: [ - (selectors) => [selectors.experiments, selectors.searchTerm, selectors.searchStatus, selectors.tab], - (experiments, searchTerm, searchStatus, tab) => { + (s) => [s.experiments, s.searchTerm, s.searchStatus, s.userFilter, s.tab], + (experiments, searchTerm, searchStatus, userFilter, tab) => { let filteredExperiments: Experiment[] = experiments if (tab === ExperimentsTabs.Archived) { @@ -125,6 +132,12 @@ export const experimentsLogic = kea([ (experiment) => getExperimentStatus(experiment) === searchStatus ) } + + if (userFilter) { + filteredExperiments = filteredExperiments.filter( + (experiment) => experiment.created_by?.uuid === userFilter + ) + } return filteredExperiments }, ], @@ -133,14 +146,9 @@ export const experimentsLogic = kea([ (hasAvailableFeature): boolean => hasAvailableFeature(AvailableFeature.EXPERIMENTATION), ], shouldShowEmptyState: [ - (s) => [s.experimentsLoading, s.filteredExperiments], - (experimentsLoading, filteredExperiments): boolean => { - return ( - filteredExperiments.length === 0 && - !experimentsLoading && - !values.searchTerm && - !values.searchStatus - ) + (s) => [s.experimentsLoading, s.experiments], + (experimentsLoading, experiments): boolean => { + return experiments.length === 0 && !experimentsLoading && !values.searchTerm && !values.searchStatus }, ], shouldShowProductIntroduction: [ diff --git a/frontend/src/scenes/feature-flags/FeatureFlag.tsx b/frontend/src/scenes/feature-flags/FeatureFlag.tsx index 3a212f4990141..6c60124c8a2ec 100644 --- a/frontend/src/scenes/feature-flags/FeatureFlag.tsx +++ b/frontend/src/scenes/feature-flags/FeatureFlag.tsx @@ -1,6 +1,7 @@ import './FeatureFlag.scss' -import { Card, Popconfirm, Radio, Skeleton } from 'antd' +import { LemonSegmentedButton } from '@posthog/lemon-ui' +import { Card, Popconfirm, Skeleton } from 'antd' import { useActions, useValues } from 'kea' import { Form, Group } from 'kea-forms' import { router } from 'kea-router' @@ -26,7 +27,6 @@ import { LemonTextArea } from 'lib/lemon-ui/LemonTextArea/LemonTextArea' import { Lettermark, LettermarkColor } from 'lib/lemon-ui/Lettermark' import { Link } from 'lib/lemon-ui/Link' import { SpinnerOverlay } from 'lib/lemon-ui/Spinner/Spinner' -import { Tooltip } from 'lib/lemon-ui/Tooltip' import { featureFlagLogic as enabledFeaturesLogic } from 'lib/logic/featureFlagLogic' import { alphabet, capitalizeFirstLetter } from 'lib/utils' import { PostHogFeature } from 'posthog-js/react' @@ -828,54 +828,41 @@ function FeatureFlagRollout({ readOnly }: { readOnly?: boolean }): JSX.Element { okText="OK" cancelText="Cancel" > - 0 - ), + ? 'This feature flag is associated with an experiment.' + : undefined, }, { - label: ( - -
- {!hasAvailableFeature(AvailableFeature.MULTIVARIATE_FLAGS) && ( - - - - )} - Multiple variants with rollout percentages (A/B test) -
- + label: !hasAvailableFeature(AvailableFeature.MULTIVARIATE_FLAGS) ? ( + + + Multiple variants with rollout percentages (A/B test) + + ) : ( + Multiple variants with rollout percentages (A/B test) ), - value: true, - disabled: !hasAvailableFeature(AvailableFeature.MULTIVARIATE_FLAGS), + value: 'multivariate', + disabledReason: !hasAvailableFeature(AvailableFeature.MULTIVARIATE_FLAGS) + ? 'This feature is not available on your current plan.' + : undefined, }, ]} - onChange={(e) => { - const { value } = e.target - if (value === false && nonEmptyVariants.length) { + onChange={(value) => { + if (value === 'boolean' && nonEmptyVariants.length) { setShowVariantDiscardWarning(true) } else { - setMultivariateEnabled(value) + setMultivariateEnabled(value === 'multivariate') focusVariantKeyField(0) } }} - value={multivariateEnabled} - optionType="button" + value={multivariateEnabled ? 'multivariate' : 'boolean'} />
diff --git a/frontend/src/scenes/feature-flags/FeatureFlags.tsx b/frontend/src/scenes/feature-flags/FeatureFlags.tsx index 5d9733aec817e..5021d0863bdc6 100644 --- a/frontend/src/scenes/feature-flags/FeatureFlags.tsx +++ b/frontend/src/scenes/feature-flags/FeatureFlags.tsx @@ -4,6 +4,7 @@ import { router } from 'kea-router' import { ActivityLog } from 'lib/components/ActivityLog/ActivityLog' import { ActivityScope } from 'lib/components/ActivityLog/humanizeActivity' import { FeatureFlagHog } from 'lib/components/hedgehogs' +import { MemberSelect } from 'lib/components/MemberSelect' import { ObjectTags } from 'lib/components/ObjectTags/ObjectTags' import { PageHeader } from 'lib/components/PageHeader' import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction' @@ -52,7 +53,7 @@ export function OverViewTab({ const { aggregationLabel } = useValues(groupsModel) const flagLogic = featureFlagsLogic({ flagPrefix }) - const { featureFlagsLoading, searchedFeatureFlags, searchTerm, uniqueCreators, filters, shouldShowEmptyState } = + const { featureFlagsLoading, searchedFeatureFlags, searchTerm, filters, shouldShowEmptyState } = useValues(flagLogic) const { updateFeatureFlag, loadFeatureFlags, setSearchTerm, setFeatureFlagsFilters } = useActions(flagLogic) const { user, hasAvailableFeature } = useValues(userLogic) @@ -274,6 +275,7 @@ export function OverViewTab({ { if (type) { if (type === 'all') { @@ -301,6 +303,7 @@ export function OverViewTab({ { if (status) { if (status === 'all') { @@ -323,22 +326,21 @@ export function OverViewTab({ Created by - { - if (user) { - if (user === 'any') { - if (filters) { - const { created_by, ...restFilters } = filters - setFeatureFlagsFilters(restFilters, true) - } - } else { - setFeatureFlagsFilters({ created_by: user }) + if (!user) { + if (filters) { + const { created_by, ...restFilters } = filters + setFeatureFlagsFilters(restFilters, true) } + } else { + setFeatureFlagsFilters({ created_by: user.id }) } }} - options={uniqueCreators} - value={filters.created_by ?? 'any'} />
diff --git a/frontend/src/scenes/feature-flags/featureFlagLogic.ts b/frontend/src/scenes/feature-flags/featureFlagLogic.ts index d793b600cbc0d..bdd313aba8281 100644 --- a/frontend/src/scenes/feature-flags/featureFlagLogic.ts +++ b/frontend/src/scenes/feature-flags/featureFlagLogic.ts @@ -6,7 +6,7 @@ import api from 'lib/api' import { convertPropertyGroupToProperties } from 'lib/components/PropertyFilters/utils' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { dayjs } from 'lib/dayjs' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { sum, toParams } from 'lib/utils' import { deleteWithUndo } from 'lib/utils/deleteWithUndo' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' diff --git a/frontend/src/scenes/feature-flags/featureFlagsLogic.ts b/frontend/src/scenes/feature-flags/featureFlagsLogic.ts index aa5df8a41afc5..6773878fd071b 100644 --- a/frontend/src/scenes/feature-flags/featureFlagsLogic.ts +++ b/frontend/src/scenes/feature-flags/featureFlagsLogic.ts @@ -3,7 +3,6 @@ import { actions, connect, events, kea, listeners, path, props, reducers, select import { loaders } from 'kea-loaders' import { actionToUrl, router, urlToAction } from 'kea-router' import api from 'lib/api' -import { LemonSelectOption } from 'lib/lemon-ui/LemonSelect' import { Scene } from 'scenes/sceneTypes' import { urls } from 'scenes/urls' @@ -25,14 +24,10 @@ export enum FeatureFlagsTab { export interface FeatureFlagsFilters { active: string - created_by: string + created_by: number type: string } -interface FeatureFlagCreators { - [id: string]: string -} - export interface FlagLogicProps { flagPrefix?: string // used to filter flags by prefix e.g. for the user interview flags } @@ -137,7 +132,7 @@ export const featureFlagsLogic = kea([ searchedFlags = searchedFlags.filter((flag) => (active === 'true' ? flag.active : !flag.active)) } if (created_by) { - searchedFlags = searchedFlags.filter((flag) => flag.created_by?.id === parseInt(created_by)) + searchedFlags = searchedFlags.filter((flag) => flag.created_by?.id === created_by) } if (type === 'boolean') { searchedFlags = searchedFlags.filter( @@ -164,24 +159,6 @@ export const featureFlagsLogic = kea([ }, ], ], - uniqueCreators: [ - (selectors) => [selectors.featureFlags], - (featureFlags) => { - const creators: FeatureFlagCreators = {} - for (const flag of featureFlags) { - if (flag.created_by) { - if (!creators[flag.created_by.id]) { - creators[flag.created_by.id] = flag.created_by.first_name - } - } - } - const response: LemonSelectOption[] = [ - { label: 'Any user', value: 'any' }, - ...Object.entries(creators).map(([id, first_name]) => ({ label: first_name, value: id })), - ] - return response - }, - ], shouldShowEmptyState: [ (s) => [s.featureFlagsLoading, s.featureFlags], (featureFlagsLoading, featureFlags): boolean => { diff --git a/frontend/src/scenes/funnels/useFunnelTooltip.tsx b/frontend/src/scenes/funnels/useFunnelTooltip.tsx index d6faa42f4e8ac..237d2e14ff4e4 100644 --- a/frontend/src/scenes/funnels/useFunnelTooltip.tsx +++ b/frontend/src/scenes/funnels/useFunnelTooltip.tsx @@ -112,9 +112,6 @@ export function useFunnelTooltip(showPersonsModal: boolean): React.RefObject{showLegend ? 'Hide' : 'Show'} legend} + label={Show legend} size="small" /> ) diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/BreakdownTag.tsx b/frontend/src/scenes/insights/filters/BreakdownFilter/BreakdownTag.tsx index da3ccc5d70224..7b551163ffe8d 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/BreakdownTag.tsx +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/BreakdownTag.tsx @@ -26,7 +26,6 @@ export function EditableBreakdownTag({ breakdown, breakdownType, isTrends }: Edi const [menuOpen, setMenuOpen] = useState(false) const logicProps = { insightProps, breakdown, breakdownType, isTrends } - const { shouldShowMenu } = useValues(breakdownTagLogic(logicProps)) const { removeBreakdown } = useActions(breakdownTagLogic(logicProps)) return ( @@ -40,13 +39,13 @@ export function EditableBreakdownTag({ breakdown, breakdownType, isTrends }: Edi breakdown={breakdown} breakdownType={breakdownType} // display remove button only if we can edit and don't have a separate menu - closable={!shouldShowMenu} + closable={false} onClose={removeBreakdown} onClick={() => { setFilterOpen(!filterOpen) }} popover={{ - overlay: shouldShowMenu ? : undefined, + overlay: , closeOnClickInside: false, onVisibilityChange: (visible) => { setMenuOpen(visible) diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/BreakdownTagMenu.tsx b/frontend/src/scenes/insights/filters/BreakdownFilter/BreakdownTagMenu.tsx index ae89f81f09966..2b03442c1200b 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/BreakdownTagMenu.tsx +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/BreakdownTagMenu.tsx @@ -4,16 +4,22 @@ import { LemonButton, LemonDivider, LemonInput, LemonSwitch } from '@posthog/lem import { useActions, useValues } from 'kea' import { IconInfo } from 'lib/lemon-ui/icons' import { Tooltip } from 'lib/lemon-ui/Tooltip' +import { insightLogic } from 'scenes/insights/insightLogic' +import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import { breakdownTagLogic } from './breakdownTagLogic' import { taxonomicBreakdownFilterLogic } from './taxonomicBreakdownFilterLogic' export const BreakdownTagMenu = (): JSX.Element => { + const { insightProps } = useValues(insightLogic) const { isHistogramable, isNormalizeable } = useValues(breakdownTagLogic) const { removeBreakdown } = useActions(breakdownTagLogic) + const { breakdown } = useValues(insightVizDataLogic(insightProps)) + const { updateBreakdown } = useActions(insightVizDataLogic(insightProps)) - const { histogramBinCount, histogramBinsUsed, breakdownFilter } = useValues(taxonomicBreakdownFilterLogic) - const { setHistogramBinCount, setHistogramBinsUsed, setNormalizeBreakdownURL } = + const { histogramBinCount, breakdownLimit, histogramBinsUsed, breakdownFilter } = + useValues(taxonomicBreakdownFilterLogic) + const { setHistogramBinCount, setBreakdownLimit, setHistogramBinsUsed, setNormalizeBreakdownURL } = useActions(taxonomicBreakdownFilterLogic) return ( @@ -50,7 +56,7 @@ export const BreakdownTagMenu = (): JSX.Element => { } /> )} - {isHistogramable && ( + {isHistogramable ? ( <> { @@ -85,6 +91,58 @@ export const BreakdownTagMenu = (): JSX.Element => { Do not bin numeric values + ) : ( + <> + + updateBreakdown({ + ...breakdown, + breakdown_hide_other_aggregation: !breakdown?.breakdown_hide_other_aggregation, + }) + } + label={ +
+ Group remaining values under "Other" + + If you have over {breakdown?.breakdown_limit ?? 25} breakdown options, the + smallest ones are aggregated under the label "Other". Use this toggle to + show/hide the "Other" option. + + } + > + + +
+ } + /> +
+ { + updateBreakdown({ breakdown_limit: breakdownLimit }) + }} + status="stealth" + active={histogramBinsUsed} + fullWidth + > + Breakdown limit:{' '} + { + setBreakdownLimit(newValue ?? 25) + }} + fullWidth={false} + className="w-20 ml-2" + type="number" + /> + +
+ )} diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/breakdownTagLogic.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/breakdownTagLogic.ts index f7dc476e70840..55260a2b168ae 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/breakdownTagLogic.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/breakdownTagLogic.ts @@ -42,10 +42,6 @@ export const breakdownTagLogic = kea([ (s) => [s.propertyDefinition], (propertyDefinition) => isURLNormalizeable(propertyDefinition?.name || ''), ], - shouldShowMenu: [ - (s) => [s.isHistogramable, s.isNormalizeable], - (isHistogramable, isNormalizeable) => isHistogramable || isNormalizeable, - ], }), listeners(({ props, actions }) => ({ removeBreakdown: () => { diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.ts index ef802a4a6495e..affadca3f1e47 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.ts @@ -42,6 +42,7 @@ export const taxonomicBreakdownFilterLogic = kea ({ breakdown }), + setBreakdownLimit: (value: number | undefined) => ({ value }), setHistogramBinsUsed: (value: boolean) => ({ value }), setHistogramBinCount: (count: number | undefined) => ({ count }), setNormalizeBreakdownURL: (normalizeBreakdownURL: boolean) => ({ @@ -55,6 +56,12 @@ export const taxonomicBreakdownFilterLogic = kea count, }, ], + localBreakdownLimit: [ + 25 as number | undefined, + { + setBreakdownLimit: (_, { value }) => value ?? 25, + }, + ], }), selectors({ breakdownFilter: [(_, p) => [p.breakdownFilter], (breakdownFilter) => breakdownFilter], @@ -91,6 +98,10 @@ export const taxonomicBreakdownFilterLogic = kea localHistogramBinCount || breakdownFilter?.breakdown_histogram_bin_count, ], + breakdownLimit: [ + (s) => [s.breakdownFilter, s.localBreakdownLimit], + (breakdownFilter, localBreakdownLimit) => localBreakdownLimit || breakdownFilter?.breakdown_limit || 25, + ], }), listeners(({ props, values }) => ({ addBreakdown: ({ breakdown, taxonomicGroup }) => { diff --git a/frontend/src/scenes/insights/insightLogic.ts b/frontend/src/scenes/insights/insightLogic.ts index 6278625757eac..9723ce8ec26b1 100644 --- a/frontend/src/scenes/insights/insightLogic.ts +++ b/frontend/src/scenes/insights/insightLogic.ts @@ -6,7 +6,7 @@ import api from 'lib/api' import { TriggerExportProps } from 'lib/components/ExportButton/exporter' import { parseProperties } from 'lib/components/PropertyFilters/utils' import { DashboardPrivilegeLevel } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { getEventNamesForAction, objectsEqual, sum, toParams } from 'lib/utils' import { eventUsageLogic, InsightEventSource } from 'lib/utils/eventUsageLogic' import { transformLegacyHiddenLegendKeys } from 'scenes/funnels/funnelUtils' diff --git a/frontend/src/scenes/insights/insightSceneLogic.tsx b/frontend/src/scenes/insights/insightSceneLogic.tsx index eb55a25fa0aa0..456203c08e521 100644 --- a/frontend/src/scenes/insights/insightSceneLogic.tsx +++ b/frontend/src/scenes/insights/insightSceneLogic.tsx @@ -1,6 +1,6 @@ import { actions, BuiltLogic, connect, kea, listeners, path, reducers, selectors, sharedListeners } from 'kea' import { actionToUrl, beforeUnload, router, urlToAction } from 'kea-router' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { eventUsageLogic, InsightEventSource } from 'lib/utils/eventUsageLogic' import { createEmptyInsight, insightLogic } from 'scenes/insights/insightLogic' import { insightLogicType } from 'scenes/insights/insightLogicType' diff --git a/frontend/src/scenes/insights/utils.tsx b/frontend/src/scenes/insights/utils.tsx index 313857cafa5d6..86d5eed8ea9b1 100644 --- a/frontend/src/scenes/insights/utils.tsx +++ b/frontend/src/scenes/insights/utils.tsx @@ -217,6 +217,16 @@ export function formatAggregationValue( // NB! Sync this with breakdown.py export const BREAKDOWN_OTHER_STRING_LABEL = '$$_posthog_breakdown_other_$$' export const BREAKDOWN_OTHER_NUMERIC_LABEL = 9007199254740991 // pow(2, 53) - 1 +export const BREAKDOWN_NULL_STRING_LABEL = '$$_posthog_breakdown_null_$$' +export const BREAKDOWN_NULL_NUMERIC_LABEL = 9007199254740990 // pow(2, 53) - 2 + +export function isOtherBreakdown(breakdown_value: string | number | null | undefined): boolean { + return breakdown_value === BREAKDOWN_OTHER_STRING_LABEL || breakdown_value === BREAKDOWN_OTHER_NUMERIC_LABEL +} + +export function isNullBreakdown(breakdown_value: string | number | null | undefined): boolean { + return breakdown_value === BREAKDOWN_NULL_STRING_LABEL || breakdown_value === BREAKDOWN_NULL_NUMERIC_LABEL +} export function formatBreakdownLabel( cohorts: CohortType[] | undefined, @@ -253,16 +263,17 @@ export function formatBreakdownLabel( } return cohorts?.filter((c) => c.id == breakdown_value)[0]?.name ?? (breakdown_value || '').toString() } else if (typeof breakdown_value == 'number') { - if (breakdown_value === BREAKDOWN_OTHER_NUMERIC_LABEL) { - return 'Other' - } - return formatPropertyValueForDisplay + return isOtherBreakdown(breakdown_value) + ? 'Other' + : isNullBreakdown(breakdown_value) + ? 'None' + : formatPropertyValueForDisplay ? formatPropertyValueForDisplay(breakdown, breakdown_value)?.toString() ?? 'None' - : breakdown_value.toString() + : String(breakdown_value) } else if (typeof breakdown_value == 'string') { - return breakdown_value === 'nan' || breakdown_value === BREAKDOWN_OTHER_STRING_LABEL + return isOtherBreakdown(breakdown_value) || breakdown_value === 'nan' ? 'Other' - : breakdown_value === '' + : isNullBreakdown(breakdown_value) || breakdown_value === '' ? 'None' : breakdown_value } else if (Array.isArray(breakdown_value)) { diff --git a/frontend/src/scenes/insights/views/BoldNumber/BoldNumber.tsx b/frontend/src/scenes/insights/views/BoldNumber/BoldNumber.tsx index d129c536a40a7..b94a691994e6c 100644 --- a/frontend/src/scenes/insights/views/BoldNumber/BoldNumber.tsx +++ b/frontend/src/scenes/insights/views/BoldNumber/BoldNumber.tsx @@ -43,9 +43,6 @@ function useBoldNumberTooltip({ useLayoutEffect(() => { tooltipEl.style.opacity = isTooltipShown ? '1' : '0' - if (isTooltipShown) { - tooltipEl.style.display = 'initial' - } const seriesResult = insightData?.result?.[0] @@ -73,10 +70,10 @@ function useBoldNumberTooltip({ useEffect(() => { const tooltipRect = tooltipEl.getBoundingClientRect() if (divRect) { - const desiredTop = window.scrollY + divRect.top - tooltipRect.height - BOLD_NUMBER_TOOLTIP_OFFSET_PX - const desiredLeft = divRect.left + divRect.width / 2 - tooltipRect.width / 2 - tooltipEl.style.top = `${Math.min(desiredTop, window.innerHeight)}px` - tooltipEl.style.left = `${Math.min(desiredLeft, window.innerWidth)}px` + tooltipEl.style.top = `${ + window.scrollY + divRect.top - tooltipRect.height - BOLD_NUMBER_TOOLTIP_OFFSET_PX + }px` + tooltipEl.style.left = `${divRect.left + divRect.width / 2 - tooltipRect.width / 2}px` } }) diff --git a/frontend/src/scenes/insights/views/LineGraph/LineGraph.tsx b/frontend/src/scenes/insights/views/LineGraph/LineGraph.tsx index 4683c4722ad45..83a60db5419c2 100644 --- a/frontend/src/scenes/insights/views/LineGraph/LineGraph.tsx +++ b/frontend/src/scenes/insights/views/LineGraph/LineGraph.tsx @@ -51,7 +51,6 @@ export function ensureTooltip(): [Root, HTMLElement] { tooltipEl = document.createElement('div') tooltipEl.id = 'InsightTooltipWrapper' tooltipEl.classList.add('InsightTooltipWrapper') - tooltipEl.style.display = 'none' document.body.appendChild(tooltipEl) } @@ -461,7 +460,6 @@ export function LineGraph_({ tooltipEl.classList.remove('above', 'below', 'no-transform') tooltipEl.classList.add(tooltip.yAlign || 'no-transform') tooltipEl.style.opacity = '1' - tooltipEl.style.display = 'initial' if (tooltip.body) { const referenceDataPoint = tooltip.dataPoints[0] // Use this point as reference to get the date @@ -549,8 +547,8 @@ export function LineGraph_({ ? chartClientLeft + tooltip.caretX - tooltipEl.clientWidth - 8 // If tooltip is too large (or close to the edge), show it to the left of the data point instead : defaultOffsetLeft - tooltipEl.style.top = Math.min(tooltipClientTop, window.innerHeight) + 'px' - tooltipEl.style.left = Math.min(tooltipClientLeft, window.innerWidth) + 'px' + tooltipEl.style.top = tooltipClientTop + 'px' + tooltipEl.style.left = tooltipClientLeft + 'px' }, }, ...(!isBar diff --git a/frontend/src/scenes/insights/views/LineGraph/PieChart.tsx b/frontend/src/scenes/insights/views/LineGraph/PieChart.tsx index 1cce2e9c26ebe..552daf3ff7830 100644 --- a/frontend/src/scenes/insights/views/LineGraph/PieChart.tsx +++ b/frontend/src/scenes/insights/views/LineGraph/PieChart.tsx @@ -203,7 +203,6 @@ export function PieChart({ tooltipEl.classList.remove('above', 'below', 'no-transform') tooltipEl.classList.add(tooltip.yAlign || 'no-transform') tooltipEl.style.opacity = '1' - tooltipEl.style.display = 'initial' if (tooltip.body) { const referenceDataPoint = tooltip.dataPoints[0] // Use this point as reference to get the date diff --git a/frontend/src/scenes/insights/views/WorldMap/WorldMap.tsx b/frontend/src/scenes/insights/views/WorldMap/WorldMap.tsx index 5085cfd419acd..a2e37bfaaded0 100644 --- a/frontend/src/scenes/insights/views/WorldMap/WorldMap.tsx +++ b/frontend/src/scenes/insights/views/WorldMap/WorldMap.tsx @@ -36,9 +36,6 @@ function useWorldMapTooltip(showPersonsModal: boolean): React.RefObject { tooltipEl.style.opacity = isTooltipShown ? '1' : '0' - if (isTooltipShown) { - tooltipEl.style.display = 'initial' - } if (tooltipCoordinates) { tooltipRoot.render( diff --git a/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrations.tsx b/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrations.tsx index 4bc890c80f38a..30ca2815d47f0 100644 --- a/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrations.tsx +++ b/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrations.tsx @@ -1,5 +1,5 @@ import { Link } from '@posthog/lemon-ui' -import { Button, Progress } from 'antd' +import { Progress } from 'antd' import { useActions, useValues } from 'kea' import { PageHeader } from 'lib/components/PageHeader' import { IconPlayCircle, IconRefresh, IconReplay } from 'lib/lemon-ui/icons' @@ -151,13 +151,13 @@ export function AsyncMigrations(): JSX.Element {
{status === AsyncMigrationStatus.NotStarted || status === AsyncMigrationStatus.FailedAtStartup ? ( - + ) : status === AsyncMigrationStatus.Starting || status === AsyncMigrationStatus.Running ? ( + return }, width: 32, }, diff --git a/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts b/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts index a3fde5d7610ee..45e2a02ed90b5 100644 --- a/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts +++ b/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts @@ -2,7 +2,7 @@ import { actions, events, kea, listeners, path, reducers, selectors } from 'kea' import { loaders } from 'kea-loaders' import { actionToUrl, urlToAction } from 'kea-router' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { isUserLoggedIn } from 'lib/utils' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic' diff --git a/frontend/src/scenes/notebooks/Nodes/NotebookNodeMention.tsx b/frontend/src/scenes/notebooks/Nodes/NotebookNodeMention.tsx index e446a8b302fce..b2dc3441b6136 100644 --- a/frontend/src/scenes/notebooks/Nodes/NotebookNodeMention.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NotebookNodeMention.tsx @@ -22,7 +22,7 @@ const Component = (props: NodeViewProps): JSX.Element => { - +
{member?.user.first_name}
{member?.user.email}
diff --git a/frontend/src/scenes/notebooks/Notebook/MentionsExtension.tsx b/frontend/src/scenes/notebooks/Notebook/MentionsExtension.tsx index d5c98de3f94ac..eafec59d4c0cb 100644 --- a/frontend/src/scenes/notebooks/Notebook/MentionsExtension.tsx +++ b/frontend/src/scenes/notebooks/Notebook/MentionsExtension.tsx @@ -121,7 +121,7 @@ export const Mentions = forwardRef(function SlashCom key={member.id} fullWidth status="primary-alt" - icon={} + icon={} active={index === selectedIndex} onClick={() => void execute(member)} > diff --git a/frontend/src/scenes/notebooks/Notebook/NotebookHistory.tsx b/frontend/src/scenes/notebooks/Notebook/NotebookHistory.tsx index 702182c5a05fe..3f72641d328e0 100644 --- a/frontend/src/scenes/notebooks/Notebook/NotebookHistory.tsx +++ b/frontend/src/scenes/notebooks/Notebook/NotebookHistory.tsx @@ -51,9 +51,11 @@ function NotebookHistoryList({ onItemClick }: { onItemClick: (logItem: ActivityL const buttonContent = ( diff --git a/frontend/src/scenes/notebooks/NotebookMenu.tsx b/frontend/src/scenes/notebooks/NotebookMenu.tsx index 1b830564f8277..7e1dd896e79ad 100644 --- a/frontend/src/scenes/notebooks/NotebookMenu.tsx +++ b/frontend/src/scenes/notebooks/NotebookMenu.tsx @@ -1,9 +1,10 @@ import './NotebookScene.scss' +import { IconClock, IconEllipsis, IconShare } from '@posthog/icons' import { LemonButton } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { router } from 'kea-router' -import { IconDelete, IconEllipsis, IconExport, IconNotification, IconShare } from 'lib/lemon-ui/icons' +import { IconDelete, IconExport } from 'lib/lemon-ui/icons' import { LemonMenu } from 'lib/lemon-ui/LemonMenu' import { urls } from 'scenes/urls' @@ -28,7 +29,7 @@ export function NotebookMenu({ shortId }: NotebookLogicProps): JSX.Element { }, { label: 'History', - icon: , + icon: , onClick: () => setShowHistory(!showHistory), }, { diff --git a/frontend/src/scenes/notebooks/NotebookSelectButton/NotebookSelectButton.tsx b/frontend/src/scenes/notebooks/NotebookSelectButton/NotebookSelectButton.tsx index 7dcf4b57e3088..48be40ae066bb 100644 --- a/frontend/src/scenes/notebooks/NotebookSelectButton/NotebookSelectButton.tsx +++ b/frontend/src/scenes/notebooks/NotebookSelectButton/NotebookSelectButton.tsx @@ -57,8 +57,7 @@ function NotebooksChoiceList(props: { sideIcon={ notebook.created_by ? ( `} /> diff --git a/frontend/src/scenes/notebooks/NotebooksTable/NotebooksTable.tsx b/frontend/src/scenes/notebooks/NotebooksTable/NotebooksTable.tsx index a122aaca2ee4b..03fbdec6bb01f 100644 --- a/frontend/src/scenes/notebooks/NotebooksTable/NotebooksTable.tsx +++ b/frontend/src/scenes/notebooks/NotebooksTable/NotebooksTable.tsx @@ -1,5 +1,6 @@ -import { LemonButton, LemonInput, LemonSelect, LemonTag } from '@posthog/lemon-ui' +import { LemonButton, LemonInput, LemonTag } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' +import { MemberSelect } from 'lib/components/MemberSelect' import { IconDelete, IconEllipsis } from 'lib/lemon-ui/icons' import { LemonBanner } from 'lib/lemon-ui/LemonBanner' import { LemonMenu } from 'lib/lemon-ui/LemonMenu' @@ -9,7 +10,6 @@ import { Link } from 'lib/lemon-ui/Link' import { useEffect } from 'react' import { ContainsTypeFilters } from 'scenes/notebooks/NotebooksTable/ContainsTypeFilter' import { DEFAULT_FILTERS, notebooksTableLogic } from 'scenes/notebooks/NotebooksTable/notebooksTableLogic' -import { membersLogic } from 'scenes/organization/membersLogic' import { urls } from 'scenes/urls' import { notebooksModel } from '~/models/notebooksModel' @@ -42,7 +42,6 @@ export function NotebooksTable(): JSX.Element { const { notebooksAndTemplates, filters, notebooksResponseLoading, notebookTemplates, sortValue, pagination } = useValues(notebooksTableLogic) const { loadNotebooks, setFilters, setSortValue } = useActions(notebooksTableLogic) - const { meFirstMembers } = useValues(membersLogic) const { selectNotebook } = useActions(notebookPanelLogic) useEffect(() => { @@ -121,20 +120,11 @@ export function NotebooksTable(): JSX.Element {
Created by: - ({ - value: x.user.uuid, - label: x.user.first_name, - })), - ]} + { - setFilters({ createdBy: v || DEFAULT_FILTERS.createdBy }) - }} - dropdownMatchSelectWidth={false} + onChange={(user) => setFilters({ createdBy: user?.uuid || DEFAULT_FILTERS.createdBy })} />
diff --git a/frontend/src/scenes/onboarding/Onboarding.tsx b/frontend/src/scenes/onboarding/Onboarding.tsx index 7392bff06e80c..d0a31a536f730 100644 --- a/frontend/src/scenes/onboarding/Onboarding.tsx +++ b/frontend/src/scenes/onboarding/Onboarding.tsx @@ -66,7 +66,7 @@ const OnboardingWrapper = ({ children }: { children: React.ReactNode }): JSX.Ele const OtherProductsStep = steps = [...steps, OtherProductsStep] } - if (featureFlags[FEATURE_FLAGS.INVITE_TEAM_MEMBER_ONBOARDING] == 'test') { + if (featureFlags[FEATURE_FLAGS.INVITE_TEAM_MEMBER_ONBOARDING] === 'test') { const inviteTeammatesStep = steps = [...steps, inviteTeammatesStep] } diff --git a/frontend/src/scenes/onboarding/onboardingLogic.tsx b/frontend/src/scenes/onboarding/onboardingLogic.tsx index 2b37a54db2329..fb4f01026d897 100644 --- a/frontend/src/scenes/onboarding/onboardingLogic.tsx +++ b/frontend/src/scenes/onboarding/onboardingLogic.tsx @@ -1,5 +1,7 @@ import { actions, connect, kea, listeners, path, props, reducers, selectors } from 'kea' import { actionToUrl, combineUrl, router, urlToAction } from 'kea-router' +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic, FeatureFlagsSet } from 'lib/logic/featureFlagLogic' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { billingLogic } from 'scenes/billing/billingLogic' import { teamLogic } from 'scenes/teamLogic' @@ -27,10 +29,12 @@ export enum OnboardingStepKey { export type AllOnboardingSteps = OnboardingStep[] export type OnboardingStep = JSX.Element -export const getProductUri = (productKey: ProductKey): string => { +export const getProductUri = (productKey: ProductKey, featureFlags: FeatureFlagsSet): string => { switch (productKey) { case ProductKey.PRODUCT_ANALYTICS: - return combineUrl(urls.events(), { onboarding_completed: true }).url + return featureFlags[FEATURE_FLAGS.REDIRECT_INGESTION_PRODUCT_ANALYTICS_ONBOARDING] === 'test' + ? combineUrl(urls.insights(), { onboarding_completed: true }).url + : combineUrl(urls.events(), { onboarding_completed: true }).url case ProductKey.SESSION_REPLAY: return urls.replay() case ProductKey.FEATURE_FLAGS: @@ -46,7 +50,7 @@ export const onboardingLogic = kea([ props({} as OnboardingLogicProps), path(['scenes', 'onboarding', 'onboardingLogic']), connect({ - values: [billingLogic, ['billing'], teamLogic, ['currentTeam']], + values: [billingLogic, ['billing'], teamLogic, ['currentTeam'], featureFlagLogic, ['featureFlags']], actions: [billingLogic, ['loadBillingSuccess'], teamLogic, ['updateCurrentTeamSuccess']], }), actions({ @@ -85,14 +89,6 @@ export const onboardingLogic = kea([ setStepKey: (_, { stepKey }) => stepKey, }, ], - onCompleteOnboardingRedirectUrl: [ - urls.default(), - { - setProductKey: (_, { productKey }) => { - return productKey ? getProductUri(productKey as ProductKey) : urls.default() - }, - }, - ], subscribedDuringOnboarding: [ false, { @@ -101,6 +97,12 @@ export const onboardingLogic = kea([ ], })), selectors({ + onCompleteOnboardingRedirectUrl: [ + (s) => [s.featureFlags, s.productKey], + (featureFlags: FeatureFlagsSet, productKey: string | null) => { + return productKey ? getProductUri(productKey as ProductKey, featureFlags) : urls.default() + }, + ], totalOnboardingSteps: [ (s) => [s.allOnboardingSteps], (allOnboardingSteps: AllOnboardingSteps) => allOnboardingSteps.length, diff --git a/frontend/src/scenes/organization/ConfirmOrganization/confirmOrganizationLogic.ts b/frontend/src/scenes/organization/ConfirmOrganization/confirmOrganizationLogic.ts index 84e328f82a51a..d375d899b8116 100644 --- a/frontend/src/scenes/organization/ConfirmOrganization/confirmOrganizationLogic.ts +++ b/frontend/src/scenes/organization/ConfirmOrganization/confirmOrganizationLogic.ts @@ -2,7 +2,7 @@ import { actions, kea, path, reducers } from 'kea' import { forms } from 'kea-forms' import { urlToAction } from 'kea-router' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import type { confirmOrganizationLogicType } from './confirmOrganizationLogicType' diff --git a/frontend/src/scenes/organization/membersLogic.tsx b/frontend/src/scenes/organization/membersLogic.tsx index 49a5ac406bc87..be1469331b99e 100644 --- a/frontend/src/scenes/organization/membersLogic.tsx +++ b/frontend/src/scenes/organization/membersLogic.tsx @@ -3,7 +3,7 @@ import { actions, connect, events, kea, listeners, path, reducers, selectors } f import { loaders } from 'kea-loaders' import api from 'lib/api' import { OrganizationMembershipLevel } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { membershipLevelToName } from 'lib/utils/permissioning' import { organizationLogic } from 'scenes/organizationLogic' import { userLogic } from 'scenes/userLogic' diff --git a/frontend/src/scenes/organizationLogic.tsx b/frontend/src/scenes/organizationLogic.tsx index 544ddd623d90d..a8d14920b2294 100644 --- a/frontend/src/scenes/organizationLogic.tsx +++ b/frontend/src/scenes/organizationLogic.tsx @@ -2,7 +2,7 @@ import { actions, afterMount, kea, listeners, path, reducers, selectors } from ' import { loaders } from 'kea-loaders' import api, { ApiConfig } from 'lib/api' import { OrganizationMembershipLevel } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { isUserLoggedIn } from 'lib/utils' import { getAppContext } from 'lib/utils/getAppContext' diff --git a/frontend/src/scenes/persons/PersonDisplay.tsx b/frontend/src/scenes/persons/PersonDisplay.tsx index 98b349859b245..f8dea0bd46fdf 100644 --- a/frontend/src/scenes/persons/PersonDisplay.tsx +++ b/frontend/src/scenes/persons/PersonDisplay.tsx @@ -30,7 +30,7 @@ export interface PersonDisplayProps { export function PersonIcon({ person, ...props -}: Pick & Omit): JSX.Element { +}: Pick & Omit): JSX.Element { const display = asDisplay(person) const email: string | undefined = useMemo(() => { @@ -41,7 +41,15 @@ export function PersonIcon({ return typeof possibleEmail === 'string' ? possibleEmail : undefined }, [person?.properties?.email]) - return + return ( + + ) } export function PersonDisplay({ diff --git a/frontend/src/scenes/persons/mergeSplitPersonLogic.ts b/frontend/src/scenes/persons/mergeSplitPersonLogic.ts index 524d7de4de293..cca79f7de906f 100644 --- a/frontend/src/scenes/persons/mergeSplitPersonLogic.ts +++ b/frontend/src/scenes/persons/mergeSplitPersonLogic.ts @@ -2,7 +2,7 @@ import { actions, connect, events, kea, key, listeners, path, props, reducers } import { loaders } from 'kea-loaders' import { router } from 'kea-router' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { PersonType } from '~/types' diff --git a/frontend/src/scenes/persons/personDeleteModalLogic.tsx b/frontend/src/scenes/persons/personDeleteModalLogic.tsx index 07da7f430e345..4bc0fe0d846da 100644 --- a/frontend/src/scenes/persons/personDeleteModalLogic.tsx +++ b/frontend/src/scenes/persons/personDeleteModalLogic.tsx @@ -1,7 +1,7 @@ import { actions, kea, path, props, reducers } from 'kea' import { loaders } from 'kea-loaders' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { toParams } from 'lib/utils' import { PersonType } from '~/types' diff --git a/frontend/src/scenes/persons/personsLogic.tsx b/frontend/src/scenes/persons/personsLogic.tsx index a8c68f528ea4b..d13bcb38aca9f 100644 --- a/frontend/src/scenes/persons/personsLogic.tsx +++ b/frontend/src/scenes/persons/personsLogic.tsx @@ -5,7 +5,7 @@ import api, { CountedPaginatedResponse } from 'lib/api' import { TriggerExportProps } from 'lib/components/ExportButton/exporter' import { convertPropertyGroupToProperties, isValidPropertyFilter } from 'lib/components/PropertyFilters/utils' import { FEATURE_FLAGS } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { toParams } from 'lib/utils' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' diff --git a/frontend/src/scenes/pipeline/appsManagementLogic.tsx b/frontend/src/scenes/pipeline/appsManagementLogic.tsx index d38948819bd85..01dddff6f771a 100644 --- a/frontend/src/scenes/pipeline/appsManagementLogic.tsx +++ b/frontend/src/scenes/pipeline/appsManagementLogic.tsx @@ -1,7 +1,7 @@ import { actions, afterMount, connect, kea, path, reducers, selectors } from 'kea' import { loaders } from 'kea-loaders' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import posthog from 'posthog-js' import { canGloballyManagePlugins, canInstallPlugins } from 'scenes/plugins/access' import { createDefaultPluginSource } from 'scenes/plugins/source/createDefaultPluginSource' diff --git a/frontend/src/scenes/plugins/edit/PluginField.tsx b/frontend/src/scenes/plugins/edit/PluginField.tsx index 48773fc765898..bfd03a84b1221 100644 --- a/frontend/src/scenes/plugins/edit/PluginField.tsx +++ b/frontend/src/scenes/plugins/edit/PluginField.tsx @@ -1,5 +1,5 @@ +import { LemonButton, LemonInput, LemonSelect } from '@posthog/lemon-ui' import { PluginConfigSchema } from '@posthog/plugin-scaffold/src/types' -import { Button, Input, Select } from 'antd' import { CodeEditor } from 'lib/components/CodeEditors' import { IconEdit } from 'lib/lemon-ui/icons' import { useState } from 'react' @@ -50,7 +50,8 @@ export function PluginField({ (value === SECRET_FIELD_VALUE || value.name === SECRET_FIELD_VALUE) ) { return ( - + ) } return fieldConfig.type === 'attachment' ? ( ) : fieldConfig.type === 'string' ? ( - + ) : fieldConfig.type === 'json' ? ( ) : fieldConfig.type === 'choice' ? ( - + { + return { label: choice, value: choice } + })} + /> ) : ( Unknown field type "{fieldConfig.type}". diff --git a/frontend/src/scenes/plugins/edit/interface-jobs/interfaceJobsLogic.ts b/frontend/src/scenes/plugins/edit/interface-jobs/interfaceJobsLogic.ts index c18eb2f98e50f..63b6cad643e2d 100644 --- a/frontend/src/scenes/plugins/edit/interface-jobs/interfaceJobsLogic.ts +++ b/frontend/src/scenes/plugins/edit/interface-jobs/interfaceJobsLogic.ts @@ -2,7 +2,7 @@ import type { FormInstance } from 'antd/lib/form/hooks/useForm.d' import { actions, events, kea, key, listeners, path, props, reducers } from 'kea' import { forms } from 'kea-forms' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { validateJson } from 'lib/utils' import { JobSpec } from '~/types' diff --git a/frontend/src/scenes/plugins/pluginsLogic.ts b/frontend/src/scenes/plugins/pluginsLogic.ts index f5d214049dec7..34d2744b13279 100644 --- a/frontend/src/scenes/plugins/pluginsLogic.ts +++ b/frontend/src/scenes/plugins/pluginsLogic.ts @@ -3,7 +3,7 @@ import { actions, afterMount, connect, kea, listeners, path, reducers, selectors import { loaders } from 'kea-loaders' import { actionToUrl, router, urlToAction } from 'kea-router' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import posthog from 'posthog-js' import { frontendAppsLogic } from 'scenes/apps/frontendAppsLogic' import { createDefaultPluginSource } from 'scenes/plugins/source/createDefaultPluginSource' diff --git a/frontend/src/scenes/plugins/source/PluginSource.tsx b/frontend/src/scenes/plugins/source/PluginSource.tsx index ad47b039462bf..11c0dccb39155 100644 --- a/frontend/src/scenes/plugins/source/PluginSource.tsx +++ b/frontend/src/scenes/plugins/source/PluginSource.tsx @@ -2,7 +2,7 @@ import './PluginSource.scss' import { useMonaco } from '@monaco-editor/react' import { Link } from '@posthog/lemon-ui' -import { Button, Skeleton } from 'antd' +import { Skeleton } from 'antd' import { useActions, useValues } from 'kea' import { Form } from 'kea-forms' import { CodeEditor } from 'lib/components/CodeEditors' @@ -81,13 +81,11 @@ export function PluginSource({ title={pluginSourceLoading ? 'Loading...' : `Edit App: ${name}`} placement={placement ?? 'left'} footer={ -
- - +
} > diff --git a/frontend/src/scenes/plugins/source/pluginSourceLogic.tsx b/frontend/src/scenes/plugins/source/pluginSourceLogic.tsx index 38f36fc30b169..4936546852c3c 100644 --- a/frontend/src/scenes/plugins/source/pluginSourceLogic.tsx +++ b/frontend/src/scenes/plugins/source/pluginSourceLogic.tsx @@ -4,7 +4,7 @@ import { loaders } from 'kea-loaders' import { beforeUnload } from 'kea-router' import api from 'lib/api' import { FormErrors } from 'lib/forms/Errors' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { validateJson } from 'lib/utils' import { frontendAppsLogic } from 'scenes/apps/frontendAppsLogic' import { pluginsLogic } from 'scenes/plugins/pluginsLogic' diff --git a/frontend/src/scenes/products/Products.tsx b/frontend/src/scenes/products/Products.tsx index eed399afba9a7..681d7a810bdd8 100644 --- a/frontend/src/scenes/products/Products.tsx +++ b/frontend/src/scenes/products/Products.tsx @@ -5,6 +5,7 @@ import { useActions, useValues } from 'kea' import { router } from 'kea-router' import { LemonCard } from 'lib/lemon-ui/LemonCard/LemonCard' import { Spinner } from 'lib/lemon-ui/Spinner' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { billingLogic } from 'scenes/billing/billingLogic' import { getProductUri } from 'scenes/onboarding/onboardingLogic' import { SceneExport } from 'scenes/sceneTypes' @@ -93,6 +94,7 @@ export function ProductCard({ className?: string }): JSX.Element { const { currentTeam } = useValues(teamLogic) + const { featureFlags } = useValues(featureFlagLogic) const onboardingCompleted = currentTeam?.has_completed_onboarding_for?.[product.type] const vertical = orientation === 'vertical' @@ -113,7 +115,7 @@ export function ProductCard({
{onboardingCompleted ? ( diff --git a/frontend/src/scenes/retention/RetentionModal.tsx b/frontend/src/scenes/retention/RetentionModal.tsx index e3f364621c592..72b0e59fb9a3e 100644 --- a/frontend/src/scenes/retention/RetentionModal.tsx +++ b/frontend/src/scenes/retention/RetentionModal.tsx @@ -25,38 +25,49 @@ export function RetentionModal(): JSX.Element | null { const { results } = useValues(retentionLogic(insightProps)) const { people, peopleLoading, peopleLoadingMore } = useValues(retentionPeopleLogic(insightProps)) const { loadMorePeople } = useActions(retentionPeopleLogic(insightProps)) - const { aggregationTargetLabel, selectedRow } = useValues(retentionModalLogic(insightProps)) + const { aggregationTargetLabel, selectedInterval, exploreUrl } = useValues(retentionModalLogic(insightProps)) const { closeModal } = useActions(retentionModalLogic(insightProps)) - if (!results || selectedRow === null) { + if (!results || selectedInterval === null) { return null } - const row = results[selectedRow] + const row = results[selectedInterval] const isEmpty = row.values[0]?.count === 0 return ( - - Close - - - void triggerExport({ - export_format: ExporterFormat.CSV, - export_context: { - path: row?.people_url, - }, - }) - } - > - Export to CSV - - +
+
+ + void triggerExport({ + export_format: ExporterFormat.CSV, + export_context: { + path: row?.people_url, + }, + }) + } + > + Download CSV + +
+ {exploreUrl && ( + { + closeModal() + }} + > + Explore + + )} +
} width={isEmpty ? undefined : '90%'} title={`${dayjs(row.date).format('MMMM D, YYYY')} Cohort`} @@ -138,9 +149,13 @@ export function RetentionModal(): JSX.Element | null { ))} - {people.next ? ( + {people.next || people.offset ? (
- + loadMorePeople(selectedInterval)} + loading={peopleLoadingMore} + > Load more {aggregationTargetLabel.plural}
diff --git a/frontend/src/scenes/retention/queries.ts b/frontend/src/scenes/retention/queries.ts new file mode 100644 index 0000000000000..3bca06cca714e --- /dev/null +++ b/frontend/src/scenes/retention/queries.ts @@ -0,0 +1,51 @@ +import { RetentionTableAppearanceType, RetentionTablePeoplePayload } from 'scenes/retention/types' + +import { query } from '~/queries/query' +import { NodeKind, PersonsQuery, RetentionQuery } from '~/queries/schema' + +export function retentionToActorsQuery(query: RetentionQuery, selectedInterval: number, offset = 0): PersonsQuery { + const group = query.aggregation_group_type_index !== undefined + const select = group ? 'group' : 'person' + return { + kind: NodeKind.PersonsQuery, + select: [select, 'appearances'], + orderBy: ['length(appearances) DESC', 'actor_id'], + source: { + kind: NodeKind.InsightPersonsQuery, + interval: selectedInterval, + source: { + ...query, + retentionFilter: { + ...query.retentionFilter, + }, + }, + }, + offset, + limit: offset ? offset * 2 : undefined, + } +} + +function appearances_1s_0s(appearances: number[], totalIntervals: number, selectedInterval: number | null): number[] { + const newTotalIntervals = totalIntervals - (selectedInterval ?? 0) + return Array.from({ length: newTotalIntervals }, (_, intervalNumber) => + appearances.includes(intervalNumber) ? 1 : 0 + ) +} + +export async function queryForActors( + retentionQuery: RetentionQuery, + selectedInterval: number, + offset: number = 0 +): Promise { + const actorsQuery = retentionToActorsQuery(retentionQuery, selectedInterval, offset) + const response = await query(actorsQuery) + const results: RetentionTableAppearanceType[] = response.results.map((row) => ({ + person: row[0], + appearances: appearances_1s_0s(row[1], retentionQuery.retentionFilter.total_intervals || 11, selectedInterval), + })) + return { + result: results, + offset: response.hasMore ? response.offset + response.limit : undefined, + missing_persons: response.missing_actors_count, + } +} diff --git a/frontend/src/scenes/retention/retentionModalLogic.ts b/frontend/src/scenes/retention/retentionModalLogic.ts index 47b1eec17c03d..4ad79ea1407c4 100644 --- a/frontend/src/scenes/retention/retentionModalLogic.ts +++ b/frontend/src/scenes/retention/retentionModalLogic.ts @@ -1,9 +1,12 @@ import { actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import { keyForInsightLogicProps } from 'scenes/insights/sharedUtils' +import { retentionToActorsQuery } from 'scenes/retention/queries' +import { urls } from 'scenes/urls' import { groupsModel, Noun } from '~/models/groupsModel' -import { isLifecycleQuery, isStickinessQuery } from '~/queries/utils' +import { DataTableNode, NodeKind, PersonsQuery, RetentionQuery } from '~/queries/schema' +import { isInsightPersonsQuery, isLifecycleQuery, isRetentionQuery, isStickinessQuery } from '~/queries/utils' import { InsightLogicProps } from '~/types' import type { retentionModalLogicType } from './retentionModalLogicType' @@ -24,7 +27,7 @@ export const retentionModalLogic = kea([ closeModal: true, })), reducers({ - selectedRow: [ + selectedInterval: [ null as number | null, { openModal: (_, { rowIndex }) => rowIndex, @@ -43,6 +46,36 @@ export const retentionModalLogic = kea([ return aggregationLabel(aggregation_group_type_index) }, ], + personsQuery: [ + (s) => [s.querySource, s.selectedInterval], + (querySource: RetentionQuery, selectedInterval): PersonsQuery | null => { + if (!querySource) { + return null + } + return retentionToActorsQuery(querySource, selectedInterval ?? 0) + }, + ], + exploreUrl: [ + (s) => [s.personsQuery], + (personsQuery): string | null => { + if (!personsQuery) { + return null + } + const query: DataTableNode = { + kind: NodeKind.DataTableNode, + source: personsQuery, + full: true, + } + if ( + isInsightPersonsQuery(personsQuery.source) && + isRetentionQuery(personsQuery.source.source) && + personsQuery.source.source.aggregation_group_type_index !== undefined + ) { + query.showPropertyFilter = false + } + return urls.insightNew(undefined, undefined, JSON.stringify(query)) + }, + ], }), listeners(({ actions }) => ({ openModal: ({ rowIndex }) => { diff --git a/frontend/src/scenes/retention/retentionPeopleLogic.ts b/frontend/src/scenes/retention/retentionPeopleLogic.ts index 2cc3b73e9a7f1..35daa1f0bd176 100644 --- a/frontend/src/scenes/retention/retentionPeopleLogic.ts +++ b/frontend/src/scenes/retention/retentionPeopleLogic.ts @@ -1,18 +1,25 @@ import { actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' import { loaders } from 'kea-loaders' import api from 'lib/api' +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { toParams } from 'lib/utils' import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import { keyForInsightLogicProps } from 'scenes/insights/sharedUtils' +import { queryForActors } from 'scenes/retention/queries' import { RetentionTablePeoplePayload } from 'scenes/retention/types' import { queryNodeToFilter } from '~/queries/nodes/InsightQuery/utils/queryNodeToFilter' +import { NodeKind } from '~/queries/schema' import { InsightLogicProps } from '~/types' import type { retentionPeopleLogicType } from './retentionPeopleLogicType' const DEFAULT_RETENTION_LOGIC_KEY = 'default_retention_key' +const hogQLInsightsRetentionFlagEnabled = (): boolean => + Boolean(featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_RETENTION]) + export const retentionPeopleLogic = kea([ props({} as InsightLogicProps), key(keyForInsightLogicProps(DEFAULT_RETENTION_LOGIC_KEY)), @@ -23,15 +30,19 @@ export const retentionPeopleLogic = kea([ })), actions(() => ({ clearPeople: true, - loadMorePeople: true, + loadMorePeople: (selectedInterval: number) => selectedInterval, loadMorePeopleSuccess: (payload: RetentionTablePeoplePayload) => ({ payload }), })), loaders(({ values }) => ({ people: { __default: {} as RetentionTablePeoplePayload, - loadPeople: async (rowIndex: number) => { - const urlParams = toParams({ ...values.apiFilters, selected_interval: rowIndex }) - return await api.get(`api/person/retention/?${urlParams}`) + loadPeople: async (selectedInterval: number) => { + if (hogQLInsightsRetentionFlagEnabled() && values.querySource?.kind === NodeKind.RetentionQuery) { + return await queryForActors(values.querySource, selectedInterval) + } + + const urlParams = toParams({ ...values.apiFilters, selected_interval: selectedInterval }) + return api.get(`api/person/retention/?${urlParams}`) }, }, })), @@ -57,12 +68,18 @@ export const retentionPeopleLogic = kea([ // clear people when changing the insight filters actions.clearPeople() }, - loadMorePeople: async () => { - if (values.people.next) { - const peopleResult: RetentionTablePeoplePayload = await api.get(values.people.next) + loadMorePeople: async (selectedInterval) => { + if (values.people.next || values.people.offset) { + let peopleResult: RetentionTablePeoplePayload + if (values.people.offset && values.querySource?.kind === NodeKind.RetentionQuery) { + peopleResult = await queryForActors(values.querySource, selectedInterval, values.people.offset) + } else { + peopleResult = await api.get(values.people.next as string) + } const newPayload: RetentionTablePeoplePayload = { result: [...(values.people.result || []), ...(peopleResult.result || [])], next: peopleResult.next, + offset: peopleResult.offset, missing_persons: (peopleResult.missing_persons || 0) + (values.people.missing_persons || 0), } actions.loadMorePeopleSuccess(newPayload) diff --git a/frontend/src/scenes/retention/types.ts b/frontend/src/scenes/retention/types.ts index 5e6a51ebf186e..c637c028fc4af 100644 --- a/frontend/src/scenes/retention/types.ts +++ b/frontend/src/scenes/retention/types.ts @@ -16,8 +16,9 @@ export interface RetentionTrendPayload { } export interface RetentionTablePeoplePayload { - next?: string - result?: RetentionTableAppearanceType[] + next?: string // Legacy support + offset?: number // Offset for HogQL queries + result?: RetentionTableAppearanceType[] // TODO: Rename to plural responses to match HogQL responses missing_persons?: number } diff --git a/frontend/src/scenes/saved-insights/SavedInsights.tsx b/frontend/src/scenes/saved-insights/SavedInsights.tsx index 1f7a7ada109b8..bf14cdad8854a 100644 --- a/frontend/src/scenes/saved-insights/SavedInsights.tsx +++ b/frontend/src/scenes/saved-insights/SavedInsights.tsx @@ -141,7 +141,7 @@ export const QUERY_TYPES_METADATA: Record = { }, [NodeKind.RetentionQuery]: { name: 'Retention', - description: 'See how many users return on subsequent days after an intial action', + description: 'See how many users return on subsequent days after an initial action', icon: IconRetention, inMenu: true, }, diff --git a/frontend/src/scenes/saved-insights/SavedInsightsFilters.tsx b/frontend/src/scenes/saved-insights/SavedInsightsFilters.tsx index e1e9e66aa111c..7859518c22366 100644 --- a/frontend/src/scenes/saved-insights/SavedInsightsFilters.tsx +++ b/frontend/src/scenes/saved-insights/SavedInsightsFilters.tsx @@ -1,9 +1,9 @@ import { IconCalendar } from '@posthog/icons' import { useActions, useValues } from 'kea' import { DateFilter } from 'lib/components/DateFilter/DateFilter' +import { MemberSelect } from 'lib/components/MemberSelect' import { LemonInput } from 'lib/lemon-ui/LemonInput/LemonInput' import { LemonSelect } from 'lib/lemon-ui/LemonSelect' -import { membersLogic } from 'scenes/organization/membersLogic' import { INSIGHT_TYPE_OPTIONS } from 'scenes/saved-insights/SavedInsights' import { savedInsightsLogic } from 'scenes/saved-insights/savedInsightsLogic' @@ -17,8 +17,6 @@ export function SavedInsightsFilters(): JSX.Element { const { tab, createdBy, insightType, dateFrom, dateTo, dashboardId, search } = filters - const { meFirstMembers } = useValues(membersLogic) - return (
Created by: - {/* TODO: Fix issues with user name order due to numbers having priority */} - ({ - value: x.user.id, - label: x.user.first_name, - })), - ]} - value={createdBy} - onChange={(v: any): void => { - setSavedInsightsFilters({ createdBy: v }) - }} - dropdownMatchSelectWidth={false} + type="secondary" + value={createdBy === 'All users' ? null : createdBy} + onChange={(user) => setSavedInsightsFilters({ createdBy: user?.id || 'All users' })} />
) : null} diff --git a/frontend/src/scenes/saved-insights/savedInsightsLogic.ts b/frontend/src/scenes/saved-insights/savedInsightsLogic.ts index cb1f40ff676d0..d2ee20db9ac43 100644 --- a/frontend/src/scenes/saved-insights/savedInsightsLogic.ts +++ b/frontend/src/scenes/saved-insights/savedInsightsLogic.ts @@ -4,7 +4,7 @@ import { actionToUrl, router, urlToAction } from 'kea-router' import api from 'lib/api' import { dayjs } from 'lib/dayjs' import { Sorting } from 'lib/lemon-ui/LemonTable' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { PaginationManual } from 'lib/lemon-ui/PaginationControl' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { objectDiffShallow, objectsEqual, toParams } from 'lib/utils' diff --git a/frontend/src/scenes/sceneTypes.ts b/frontend/src/scenes/sceneTypes.ts index 7ce70ef4647df..4604a64a697da 100644 --- a/frontend/src/scenes/sceneTypes.ts +++ b/frontend/src/scenes/sceneTypes.ts @@ -133,4 +133,7 @@ export interface SceneConfig { organizationBased?: boolean /** Route requires project access (used e.g. by breadcrumbs). `true` implies also `organizationBased` */ projectBased?: boolean + + /** Default docs path - what the docs side panel will open by default if this scene is active */ + defaultDocsPath?: string } diff --git a/frontend/src/scenes/scenes.ts b/frontend/src/scenes/scenes.ts index 81a660329aa6e..4ebd425f741f0 100644 --- a/frontend/src/scenes/scenes.ts +++ b/frontend/src/scenes/scenes.ts @@ -1,6 +1,6 @@ import { combineUrl } from 'kea-router' import { dayjs } from 'lib/dayjs' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { getDefaultEventsSceneQuery } from 'scenes/events/defaults' import { LoadedScene, Params, Scene, SceneConfig } from 'scenes/sceneTypes' import { urls } from 'scenes/urls' @@ -46,38 +46,48 @@ export const sceneConfigurations: Record = { [Scene.Dashboards]: { projectBased: true, name: 'Dashboards', + defaultDocsPath: '/docs/product-analytics/dashboards', }, [Scene.Dashboard]: { projectBased: true, + defaultDocsPath: '/docs/product-analytics/dashboards', }, + [Scene.Insight]: { projectBased: true, name: 'Insights', + defaultDocsPath: '/docs/product-analytics/insights', }, [Scene.WebAnalytics]: { projectBased: true, name: 'Web analytics', layout: 'app-container', + defaultDocsPath: '/docs/web-analytics', }, [Scene.Cohort]: { projectBased: true, name: 'Cohort', + defaultDocsPath: '/docs/data/cohorts', }, [Scene.Events]: { projectBased: true, name: 'Event explorer', + defaultDocsPath: '/docs/data/events', }, [Scene.BatchExports]: { projectBased: true, name: 'Batch exports', + defaultDocsPath: '/docs/cdp/batch-exports', }, [Scene.BatchExportEdit]: { projectBased: true, name: 'Edit batch export', + defaultDocsPath: '/docs/cdp/batch-exports', }, [Scene.BatchExport]: { projectBased: true, name: 'Batch export', + defaultDocsPath: '/docs/cdp/batch-exports', }, [Scene.DataManagement]: { projectBased: true, @@ -86,6 +96,7 @@ export const sceneConfigurations: Record = { [Scene.EventDefinition]: { projectBased: true, name: 'Data management', + defaultDocsPath: '/docs/data/events', }, [Scene.PropertyDefinition]: { projectBased: true, @@ -94,111 +105,139 @@ export const sceneConfigurations: Record = { [Scene.Replay]: { projectBased: true, name: 'Session replay', + defaultDocsPath: '/docs/session-replay', }, [Scene.ReplaySingle]: { projectBased: true, name: 'Replay recording', + defaultDocsPath: '/docs/session-replay', }, [Scene.ReplayPlaylist]: { projectBased: true, name: 'Replay playlist', + defaultDocsPath: '/docs/session-replay', }, [Scene.Person]: { projectBased: true, name: 'Person', + defaultDocsPath: '/docs/session-replay', }, [Scene.PersonsManagement]: { projectBased: true, name: 'People & groups', + defaultDocsPath: '/docs/data/persons', }, [Scene.Action]: { projectBased: true, name: 'Action', + defaultDocsPath: '/docs/data/actions', }, [Scene.Group]: { projectBased: true, name: 'People & groups', + defaultDocsPath: '/docs/product-analytics/group-analytics', }, [Scene.Pipeline]: { projectBased: true, name: 'Pipeline', + defaultDocsPath: '/docs/cdp', }, [Scene.PipelineApp]: { projectBased: true, name: 'Pipeline app', + defaultDocsPath: '/docs/cdp', }, [Scene.Experiments]: { projectBased: true, name: 'A/B testing', + defaultDocsPath: '/docs/experiments', }, [Scene.Experiment]: { projectBased: true, name: 'Experiment', + defaultDocsPath: '/docs/experiments/creating-an-experiment', }, [Scene.FeatureFlags]: { projectBased: true, name: 'Feature flags', + defaultDocsPath: '/docs/feature-flags', }, [Scene.FeatureFlag]: { projectBased: true, + defaultDocsPath: '/docs/feature-flags/creating-feature-flags', }, [Scene.Surveys]: { projectBased: true, name: 'Surveys', + defaultDocsPath: '/docs/feature-flags/creating-feature-flags', }, [Scene.Survey]: { projectBased: true, name: 'Survey', + defaultDocsPath: '/docs/surveys', }, [Scene.SurveyTemplates]: { projectBased: true, name: 'New survey', + defaultDocsPath: '/docs/surveys/creating-surveys', }, [Scene.DataWarehouse]: { projectBased: true, name: 'Data warehouse', + defaultDocsPath: '/docs/feature-flags/creating-feature-flags', }, [Scene.DataWarehousePosthog]: { projectBased: true, name: 'Data warehouse', + defaultDocsPath: '/docs/data-warehouse', }, [Scene.DataWarehouseExternal]: { projectBased: true, name: 'Data warehouse', + defaultDocsPath: '/docs/data-warehouse/setup', }, [Scene.DataWarehouseSavedQueries]: { projectBased: true, name: 'Data warehouse', + defaultDocsPath: '/docs/data-warehouse/view', }, [Scene.DataWarehouseSettings]: { projectBased: true, name: 'Data warehouse settings', + defaultDocsPath: '/docs/data-warehouse', }, [Scene.DataWarehouseTable]: { projectBased: true, name: 'Data warehouse table', + defaultDocsPath: '/docs/data-warehouse', }, [Scene.EarlyAccessFeatures]: { projectBased: true, + defaultDocsPath: '/docs/data-warehouse', }, [Scene.EarlyAccessFeature]: { projectBased: true, + defaultDocsPath: '/docs/feature-flags/early-access-feature-management', }, [Scene.Apps]: { projectBased: true, name: 'Apps', + defaultDocsPath: '/docs/cdp', }, [Scene.FrontendAppScene]: { projectBased: true, name: 'App', + defaultDocsPath: '/docs/cdp', }, [Scene.AppMetrics]: { projectBased: true, name: 'Apps', + defaultDocsPath: '/docs/cdp', }, [Scene.SavedInsights]: { projectBased: true, name: 'Product analytics', + defaultDocsPath: '/docs/product-analytics', }, [Scene.ProjectHomepage]: { projectBased: true, @@ -218,6 +257,7 @@ export const sceneConfigurations: Record = { [Scene.ToolbarLaunch]: { projectBased: true, name: 'Launch toolbar', + defaultDocsPath: '/docs/toolbar', }, [Scene.Site]: { projectBased: true, @@ -227,14 +267,17 @@ export const sceneConfigurations: Record = { // Organization-based routes [Scene.OrganizationCreateFirst]: { name: 'Organization creation', + defaultDocsPath: '/docs/data/organizations-and-projects', }, [Scene.OrganizationCreationConfirm]: { name: 'Confirm organization creation', onlyUnauthenticated: true, + defaultDocsPath: '/docs/data/organizations-and-projects', }, [Scene.ProjectCreateFirst]: { name: 'Project creation', organizationBased: true, + defaultDocsPath: '/docs/data/organizations-and-projects', }, // Onboarding/setup routes [Scene.Login]: { @@ -274,6 +317,7 @@ export const sceneConfigurations: Record = { [Scene.Billing]: { hideProjectNotice: true, organizationBased: true, + defaultDocsPath: '/pricing', }, [Scene.Unsubscribe]: { allowUnauthenticated: true, @@ -295,15 +339,18 @@ export const sceneConfigurations: Record = { hideProjectNotice: true, // Currently doesn't render well... name: 'Notebook', layout: 'app-raw', + defaultDocsPath: '/blog/introducing-notebooks', }, [Scene.Notebooks]: { projectBased: true, name: 'Notebooks', + defaultDocsPath: '/blog/introducing-notebooks', }, [Scene.Canvas]: { projectBased: true, name: 'Canvas', layout: 'app-raw', + defaultDocsPath: '/blog/introducing-notebooks', }, [Scene.Settings]: { projectBased: true, diff --git a/frontend/src/scenes/session-recordings/player/PlayerFrame.scss b/frontend/src/scenes/session-recordings/player/PlayerFrame.scss index 9682ebb04bf9e..924b067190978 100644 --- a/frontend/src/scenes/session-recordings/player/PlayerFrame.scss +++ b/frontend/src/scenes/session-recordings/player/PlayerFrame.scss @@ -10,6 +10,10 @@ overflow: hidden; background-color: var(--bg-charcoal); + .posthog-3000 & { + background-color: var(--bg-3000-dark); + } + .PlayerFrame__content { position: absolute; diff --git a/frontend/src/scenes/session-recordings/player/utils/playerUtils.ts b/frontend/src/scenes/session-recordings/player/utils/playerUtils.ts index 7fe1d5ecedceb..5ea42f08b6d17 100644 --- a/frontend/src/scenes/session-recordings/player/utils/playerUtils.ts +++ b/frontend/src/scenes/session-recordings/player/utils/playerUtils.ts @@ -1,7 +1,7 @@ import { router } from 'kea-router' import api from 'lib/api' import { ExpandableConfig } from 'lib/lemon-ui/LemonTable' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { MouseEvent as ReactMouseEvent, TouchEvent as ReactTouchEvent } from 'react' import { urls } from 'scenes/urls' diff --git a/frontend/src/scenes/session-recordings/playlist/playlistUtils.ts b/frontend/src/scenes/session-recordings/playlist/playlistUtils.ts index c12e3a950cdd8..a1e8ac8673d09 100644 --- a/frontend/src/scenes/session-recordings/playlist/playlistUtils.ts +++ b/frontend/src/scenes/session-recordings/playlist/playlistUtils.ts @@ -1,7 +1,7 @@ import { router } from 'kea-router' import api from 'lib/api' import { convertPropertyGroupToProperties } from 'lib/components/PropertyFilters/utils' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { getKeyMapping } from 'lib/taxonomy' import { genericOperatorMap } from 'lib/utils' import { deleteWithUndo } from 'lib/utils/deleteWithUndo' diff --git a/frontend/src/scenes/session-recordings/saved-playlists/SavedSessionRecordingPlaylists.tsx b/frontend/src/scenes/session-recordings/saved-playlists/SavedSessionRecordingPlaylists.tsx index 69733a2826df3..aafd93bbb834a 100644 --- a/frontend/src/scenes/session-recordings/saved-playlists/SavedSessionRecordingPlaylists.tsx +++ b/frontend/src/scenes/session-recordings/saved-playlists/SavedSessionRecordingPlaylists.tsx @@ -1,13 +1,13 @@ import { TZLabel } from '@posthog/apps-common' -import { LemonButton, LemonDivider, LemonInput, LemonSelect, LemonTable, Link } from '@posthog/lemon-ui' +import { LemonButton, LemonDivider, LemonInput, LemonTable, Link } from '@posthog/lemon-ui' import clsx from 'clsx' import { useActions, useValues } from 'kea' import { DateFilter } from 'lib/components/DateFilter/DateFilter' +import { MemberSelect } from 'lib/components/MemberSelect' import { IconCalendar, IconPinFilled, IconPinOutline } from 'lib/lemon-ui/icons' import { More } from 'lib/lemon-ui/LemonButton/More' import { LemonTableColumn, LemonTableColumns } from 'lib/lemon-ui/LemonTable' import { createdByColumn } from 'lib/lemon-ui/LemonTable/columnUtils' -import { membersLogic } from 'scenes/organization/membersLogic' import { SavedSessionRecordingPlaylistsEmptyState } from 'scenes/session-recordings/saved-playlists/SavedSessionRecordingPlaylistsEmptyState' import { urls } from 'scenes/urls' @@ -40,7 +40,6 @@ export function SavedSessionRecordingPlaylists({ tab }: SavedSessionRecordingPla const logic = savedSessionRecordingPlaylistsLogic({ tab }) const { playlists, playlistsLoading, filters, sorting, pagination } = useValues(logic) const { setSavedPlaylistsFilters, updatePlaylist, duplicatePlaylist, deletePlaylist } = useActions(logic) - const { meFirstMembers } = useValues(membersLogic) const columns: LemonTableColumns = [ { @@ -159,20 +158,11 @@ export function SavedSessionRecordingPlaylists({ tab }: SavedSessionRecordingPla
Created by: - ({ - value: x.user.id, - label: x.user.first_name, - })), - ]} - value={filters.createdBy} - onChange={(v: any): void => { - setSavedPlaylistsFilters({ createdBy: v }) - }} - dropdownMatchSelectWidth={false} + type="secondary" + value={filters.createdBy === 'All users' ? null : filters.createdBy} + onChange={(user) => setSavedPlaylistsFilters({ createdBy: user?.id || 'All users' })} />
diff --git a/frontend/src/scenes/settings/SettingsMap.tsx b/frontend/src/scenes/settings/SettingsMap.tsx index 579e2976a1fde..966e43e9c8dd3 100644 --- a/frontend/src/scenes/settings/SettingsMap.tsx +++ b/frontend/src/scenes/settings/SettingsMap.tsx @@ -34,6 +34,7 @@ import { SettingSection } from './types' import { ChangePassword } from './user/ChangePassword' import { OptOutCapture } from './user/OptOutCapture' import { PersonalAPIKeys } from './user/PersonalAPIKeys' +import { ThemeSwitcher } from './user/ThemeSwitcher' import { TwoFactorAuthentication } from './user/TwoFactorAuthentication' import { UpdateEmailPreferences } from './user/UpdateEmailPreferences' import { UserDetails } from './user/UserDetails' @@ -343,9 +344,14 @@ export const SettingsMap: SettingSection[] = [ }, { level: 'user', - id: 'user-notifications', - title: 'Notifications', + id: 'user-customization', + title: 'Customization', settings: [ + { + id: 'theme', + title: 'Theme', + component: , + }, { id: 'notifications', title: 'Notifications', diff --git a/frontend/src/scenes/settings/organization/Invites.tsx b/frontend/src/scenes/settings/organization/Invites.tsx index 132184c745226..823f24997fb14 100644 --- a/frontend/src/scenes/settings/organization/Invites.tsx +++ b/frontend/src/scenes/settings/organization/Invites.tsx @@ -62,7 +62,7 @@ export function Invites(): JSX.Element { { key: 'user_profile_picture', render: function ProfilePictureRender(_, invite) { - return + return }, width: 32, }, diff --git a/frontend/src/scenes/settings/organization/Members.tsx b/frontend/src/scenes/settings/organization/Members.tsx index c892bbd7dc4e9..ae4a31509a537 100644 --- a/frontend/src/scenes/settings/organization/Members.tsx +++ b/frontend/src/scenes/settings/organization/Members.tsx @@ -10,6 +10,7 @@ import { LemonTable, LemonTableColumns } from 'lib/lemon-ui/LemonTable' import { LemonTag } from 'lib/lemon-ui/LemonTag/LemonTag' import { ProfilePicture } from 'lib/lemon-ui/ProfilePicture' import { Tooltip } from 'lib/lemon-ui/Tooltip' +import { fullName } from 'lib/utils' import { getReasonForAccessLevelChangeProhibition, membershipLevelToName, @@ -68,7 +69,7 @@ function ActionsComponent(_: any, member: OrganizationMemberType): JSX.Element | } if (listLevel === OrganizationMembershipLevel.Owner) { LemonDialog.open({ - title: `Transfer organization ownership to ${member.user.first_name}?`, + title: `Transfer organization ownership to ${fullName(member.user)}?`, description: `You will no longer be the owner of ${user.organization?.name}. After the transfer you will become an administrator.`, primaryButton: { status: 'danger', @@ -109,7 +110,7 @@ function ActionsComponent(_: any, member: OrganizationMemberType): JSX.Element | title: `${ member.user.uuid == user.uuid ? 'Leave' - : `Remove ${member.user.first_name} from` + : `Remove ${fullName(member.user)} from` } organization ${user.organization?.name}?`, primaryButton: { children: member.user.uuid == user.uuid ? 'Leave' : 'Remove', @@ -150,16 +151,16 @@ export function Members(): JSX.Element | null { { key: 'user_profile_picture', render: function ProfilePictureRender(_, member) { - return + return }, width: 32, }, { title: 'Name', - key: 'user_first_name', + key: 'user_name', render: (_, member) => - member.user.uuid == user.uuid ? `${member.user.first_name} (me)` : member.user.first_name, - sorter: (a, b) => a.user.first_name.localeCompare(b.user.first_name), + member.user.uuid == user.uuid ? `${fullName(member.user)} (me)` : fullName(member.user), + sorter: (a, b) => fullName(a.user).localeCompare(fullName(b.user)), }, { title: 'Email', diff --git a/frontend/src/scenes/settings/organization/Permissions/Roles/CreateRoleModal.tsx b/frontend/src/scenes/settings/organization/Permissions/Roles/CreateRoleModal.tsx index 3311cc1833117..fc2d6571dbf2d 100644 --- a/frontend/src/scenes/settings/organization/Permissions/Roles/CreateRoleModal.tsx +++ b/frontend/src/scenes/settings/organization/Permissions/Roles/CreateRoleModal.tsx @@ -154,7 +154,7 @@ function MemberRow({ return (
- + {isAdminOrOwner && deleteMember && ( } diff --git a/frontend/src/scenes/settings/organization/VerifiedDomains/verifiedDomainsLogic.ts b/frontend/src/scenes/settings/organization/VerifiedDomains/verifiedDomainsLogic.ts index fbc0a829bc5ba..00778b3e82875 100644 --- a/frontend/src/scenes/settings/organization/VerifiedDomains/verifiedDomainsLogic.ts +++ b/frontend/src/scenes/settings/organization/VerifiedDomains/verifiedDomainsLogic.ts @@ -3,7 +3,7 @@ import { forms } from 'kea-forms' import { loaders } from 'kea-loaders' import api from 'lib/api' import { SECURE_URL_REGEX } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { organizationLogic } from 'scenes/organizationLogic' import { AvailableFeature, OrganizationDomainType } from '~/types' diff --git a/frontend/src/scenes/settings/organization/inviteLogic.ts b/frontend/src/scenes/settings/organization/inviteLogic.ts index 32865643d3a21..0dabd3ec32807 100644 --- a/frontend/src/scenes/settings/organization/inviteLogic.ts +++ b/frontend/src/scenes/settings/organization/inviteLogic.ts @@ -2,7 +2,7 @@ import { actions, connect, events, kea, listeners, path, reducers, selectors } f import { loaders } from 'kea-loaders' import { router, urlToAction } from 'kea-router' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { organizationLogic } from 'scenes/organizationLogic' import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic' diff --git a/frontend/src/scenes/settings/organization/invitesLogic.tsx b/frontend/src/scenes/settings/organization/invitesLogic.tsx index 0cb80e1dacca3..4a53c9b5846c8 100644 --- a/frontend/src/scenes/settings/organization/invitesLogic.tsx +++ b/frontend/src/scenes/settings/organization/invitesLogic.tsx @@ -1,7 +1,7 @@ import { events, kea, listeners, path } from 'kea' import { loaders } from 'kea-loaders' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic' diff --git a/frontend/src/scenes/settings/project/ProjectAccessControl.tsx b/frontend/src/scenes/settings/project/ProjectAccessControl.tsx index b34d3a52fd94c..ff18dd636e143 100644 --- a/frontend/src/scenes/settings/project/ProjectAccessControl.tsx +++ b/frontend/src/scenes/settings/project/ProjectAccessControl.tsx @@ -140,7 +140,7 @@ export function ProjectTeamMembers(): JSX.Element | null { { key: 'user_profile_picture', render: function ProfilePictureRender(_, member) { - return + return }, width: 32, }, diff --git a/frontend/src/scenes/settings/project/teamMembersLogic.tsx b/frontend/src/scenes/settings/project/teamMembersLogic.tsx index f9457bca55b00..80a366f2e2f31 100644 --- a/frontend/src/scenes/settings/project/teamMembersLogic.tsx +++ b/frontend/src/scenes/settings/project/teamMembersLogic.tsx @@ -3,7 +3,7 @@ import { forms } from 'kea-forms' import { loaders } from 'kea-loaders' import api from 'lib/api' import { OrganizationMembershipLevel, TeamMembershipLevel } from 'lib/constants' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { membershipLevelToName } from 'lib/utils/permissioning' import { membersLogic } from 'scenes/organization/membersLogic' diff --git a/frontend/src/scenes/settings/project/webhookIntegrationLogic.ts b/frontend/src/scenes/settings/project/webhookIntegrationLogic.ts index b9df2b8de3b72..f3156eb963cba 100644 --- a/frontend/src/scenes/settings/project/webhookIntegrationLogic.ts +++ b/frontend/src/scenes/settings/project/webhookIntegrationLogic.ts @@ -1,7 +1,7 @@ import { kea, listeners, path, selectors } from 'kea' import { loaders } from 'kea-loaders' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { capitalizeFirstLetter } from 'lib/utils' import { teamLogic } from 'scenes/teamLogic' diff --git a/frontend/src/scenes/settings/types.ts b/frontend/src/scenes/settings/types.ts index 01ed3e2842ff0..6dad2d9f75193 100644 --- a/frontend/src/scenes/settings/types.ts +++ b/frontend/src/scenes/settings/types.ts @@ -32,7 +32,7 @@ export type SettingSectionId = | 'organization-danger-zone' | 'user-profile' | 'user-api-keys' - | 'user-notifications' + | 'user-customization' export type SettingId = | 'display-name' @@ -72,6 +72,7 @@ export type SettingId = | 'personal-api-keys' | 'notifications' | 'optout' + | 'theme' export type Setting = { id: SettingId diff --git a/frontend/src/scenes/settings/user/ThemeSwitcher.tsx b/frontend/src/scenes/settings/user/ThemeSwitcher.tsx new file mode 100644 index 0000000000000..a96feb61c8f48 --- /dev/null +++ b/frontend/src/scenes/settings/user/ThemeSwitcher.tsx @@ -0,0 +1,40 @@ +import { IconDay, IconLaptop, IconNight } from '@posthog/icons' +import { LemonSelect, LemonSelectProps } from '@posthog/lemon-ui' +import { useActions, useValues } from 'kea' +import { userLogic } from 'scenes/userLogic' + +export function ThemeSwitcher({ + onlyLabel, + ...props +}: Partial> & { onlyLabel?: boolean }): JSX.Element { + const { themeMode } = useValues(userLogic) + const { updateUser } = useActions(userLogic) + + return ( + , value: 'system', label: `Sync with system` }, + { icon: , value: 'light', label: 'Light mode' }, + { icon: , value: 'dark', label: 'Dark mode' }, + ]} + value={themeMode} + renderButtonContent={(leaf) => { + const labelText = leaf ? leaf.label : 'Sync with system' + return onlyLabel ? ( + labelText + ) : ( + <> + + Color theme + {leaf ? leaf.label : 'Sync with system'} + + + ) + }} + onChange={(value) => updateUser({ theme_mode: value })} + dropdownPlacement="right-start" + dropdownMatchSelectWidth={false} + {...props} + /> + ) +} diff --git a/frontend/src/scenes/settings/user/personalAPIKeysLogic.ts b/frontend/src/scenes/settings/user/personalAPIKeysLogic.ts index 7b04396e5079d..6e743b5329f7f 100644 --- a/frontend/src/scenes/settings/user/personalAPIKeysLogic.ts +++ b/frontend/src/scenes/settings/user/personalAPIKeysLogic.ts @@ -1,7 +1,7 @@ import { kea, listeners, path } from 'kea' import { loaders } from 'kea-loaders' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { copyToClipboard } from 'lib/utils/copyToClipboard' import { PersonalAPIKeyType } from '~/types' diff --git a/frontend/src/scenes/surveys/SurveyAppearance.tsx b/frontend/src/scenes/surveys/SurveyAppearance.tsx index ef0d568f3ceb4..dbf5349ad3f8f 100644 --- a/frontend/src/scenes/surveys/SurveyAppearance.tsx +++ b/frontend/src/scenes/surveys/SurveyAppearance.tsx @@ -56,7 +56,7 @@ interface ButtonProps { children: React.ReactNode } -const Button = ({ +const SurveyButton = ({ link, type, onSubmit, @@ -351,7 +351,7 @@ export function BaseAppearance({
- +
{!preview && !appearance.whiteLabel && ( @@ -572,13 +572,13 @@ export function SurveyRatingAppearance({
- +
{!preview && !appearance.whiteLabel && ( @@ -739,9 +739,13 @@ export function SurveyMultipleChoiceAppearance({
- +
{!preview && !appearance.whiteLabel && ( @@ -797,9 +801,9 @@ export function SurveyThankYou({ appearance }: { appearance: SurveyAppearanceTyp className="thank-you-message-body" dangerouslySetInnerHTML={{ __html: sanitizeHTML(appearance?.thankYouMessageDescription || '') }} /> - + {!appearance.whiteLabel && ( Survey by {posthogLogoSVG} diff --git a/frontend/src/scenes/surveys/Surveys.tsx b/frontend/src/scenes/surveys/Surveys.tsx index 52b79940c00b8..e8f0f631cac2f 100644 --- a/frontend/src/scenes/surveys/Surveys.tsx +++ b/frontend/src/scenes/surveys/Surveys.tsx @@ -11,6 +11,7 @@ import { } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { router } from 'kea-router' +import { MemberSelect } from 'lib/components/MemberSelect' import { PageHeader } from 'lib/components/PageHeader' import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction' import { VersionCheckerBanner } from 'lib/components/VersionChecker/VersionCheckerBanner' @@ -53,7 +54,6 @@ export function Surveys(): JSX.Element { surveysResponsesCountLoading, searchTerm, filters, - uniqueCreators, showSurveysDisabledBanner, } = useValues(surveysLogic) @@ -170,6 +170,7 @@ export function Surveys(): JSX.Element { onChange={(status) => { setSurveysFilters({ status }) }} + size="small" options={[ { label: 'Any', value: 'any' }, { label: 'Draft', value: 'draft' }, @@ -181,12 +182,12 @@ export function Surveys(): JSX.Element { Created by - { - setSurveysFilters({ created_by: user }) - }} - options={uniqueCreators} - value={filters.created_by} + setSurveysFilters({ created_by: user?.id })} />
diff --git a/frontend/src/scenes/surveys/surveysLogic.tsx b/frontend/src/scenes/surveys/surveysLogic.tsx index c0a0dc2717dc5..d31ffd5ac7a0d 100644 --- a/frontend/src/scenes/surveys/surveysLogic.tsx +++ b/frontend/src/scenes/surveys/surveysLogic.tsx @@ -5,7 +5,6 @@ import { loaders } from 'kea-loaders' import { router } from 'kea-router' import api from 'lib/api' import { FEATURE_FLAGS } from 'lib/constants' -import { LemonSelectOption } from 'lib/lemon-ui/LemonSelect' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { Scene } from 'scenes/sceneTypes' import { teamLogic } from 'scenes/teamLogic' @@ -27,14 +26,10 @@ export function getSurveyStatus(survey: Survey): ProgressStatus { export interface SurveysFilters { status: string - created_by: string + created_by: null | number archived: boolean } -interface SurveysCreators { - [id: string]: string -} - export const surveysLogic = kea([ path(['scenes', 'surveys', 'surveysLogic']), connect(() => ({ @@ -84,7 +79,7 @@ export const surveysLogic = kea([ { archived: false, status: 'any', - created_by: 'any', + created_by: null, } as Partial, { setSurveysFilters: (state, { filters }) => { @@ -133,10 +128,8 @@ export const surveysLogic = kea([ if (status !== 'any') { searchedSurveys = searchedSurveys.filter((survey) => getSurveyStatus(survey) === status) } - if (created_by !== 'any') { - searchedSurveys = searchedSurveys.filter( - (survey) => survey.created_by?.id === (created_by ? parseInt(created_by) : '') - ) + if (created_by) { + searchedSurveys = searchedSurveys.filter((survey) => survey.created_by?.id === created_by) } if (archived) { @@ -158,24 +151,6 @@ export const surveysLogic = kea([ }, ], ], - uniqueCreators: [ - (selectors) => [selectors.surveys], - (surveys) => { - const creators: SurveysCreators = {} - for (const survey of surveys) { - if (survey.created_by) { - if (!creators[survey.created_by.id]) { - creators[survey.created_by.id] = survey.created_by.first_name - } - } - } - const response: LemonSelectOption[] = [ - { label: 'Any user', value: 'any' }, - ...Object.entries(creators).map(([id, first_name]) => ({ label: first_name, value: id })), - ] - return response - }, - ], payGateFlagOn: [(s) => [s.featureFlags], (featureFlags) => featureFlags[FEATURE_FLAGS.SURVEYS_PAYGATES]], whitelabelAvailable: [ (s) => [s.hasAvailableFeature], diff --git a/frontend/src/scenes/teamLogic.tsx b/frontend/src/scenes/teamLogic.tsx index 87a87e7f7cbca..b9f0b5ebc3664 100644 --- a/frontend/src/scenes/teamLogic.tsx +++ b/frontend/src/scenes/teamLogic.tsx @@ -3,7 +3,7 @@ import { loaders } from 'kea-loaders' import api, { ApiConfig } from 'lib/api' import { OrganizationMembershipLevel } from 'lib/constants' import { IconSwapHoriz } from 'lib/lemon-ui/icons' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { getPropertyLabel } from 'lib/taxonomy' import { identifierToHuman, isUserLoggedIn, resolveWebhookService } from 'lib/utils' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' diff --git a/frontend/src/scenes/trends/Trends.tsx b/frontend/src/scenes/trends/Trends.tsx index 1afc7a25fce3c..6c60866d8948d 100644 --- a/frontend/src/scenes/trends/Trends.tsx +++ b/frontend/src/scenes/trends/Trends.tsx @@ -67,9 +67,9 @@ export function TrendInsight({ view, context }: Props): JSX.Element { {display !== ChartDisplayType.WorldMap && // the world map doesn't need this cta breakdown && (hasBreakdownOther || loadMoreBreakdownUrl) && ( -
-
- For readability, not all breakdown values are displayed. Click below to load them. +
+
+ For readability, not all breakdown values are displayed. Click below to load more.
([ } }, })), + + propsChanged(({ props, actions }, oldProps) => { + if (props.url !== oldProps.url) { + actions.loadActors({ query: props.query, url: props.url, clear: true }) + } + }), ]) diff --git a/frontend/src/scenes/trends/trendsDataLogic.ts b/frontend/src/scenes/trends/trendsDataLogic.ts index f020b526fd4f3..df1638072c93b 100644 --- a/frontend/src/scenes/trends/trendsDataLogic.ts +++ b/frontend/src/scenes/trends/trendsDataLogic.ts @@ -3,7 +3,7 @@ import api from 'lib/api' import { dayjs } from 'lib/dayjs' import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import { keyForInsightLogicProps } from 'scenes/insights/sharedUtils' -import { BREAKDOWN_OTHER_NUMERIC_LABEL, BREAKDOWN_OTHER_STRING_LABEL } from 'scenes/insights/utils' +import { isOtherBreakdown } from 'scenes/insights/utils' import { EntityNode } from '~/queries/schema' import { @@ -104,14 +104,7 @@ export const trendsDataLogic = kea([ return false } const results = insightData.result ?? insightData.results - return !!( - Array.isArray(results) && - results.find( - (r) => - r.breakdown_value === BREAKDOWN_OTHER_STRING_LABEL || - r.breakdown_value === BREAKDOWN_OTHER_NUMERIC_LABEL - ) - ) + return !!(Array.isArray(results) && results.find((r) => isOtherBreakdown(r.breakdown_value))) }, ], diff --git a/frontend/src/scenes/urls.ts b/frontend/src/scenes/urls.ts index 26452f9e94f5f..bd5cbe5939aec 100644 --- a/frontend/src/scenes/urls.ts +++ b/frontend/src/scenes/urls.ts @@ -64,6 +64,7 @@ export const urls = { `/batch_exports/${id}` + (params ? `?${toParams(params)}` : ''), batchExportEdit: (id: string): string => `/batch_exports/${id}/edit`, ingestionWarnings: (): string => '/data-management/ingestion-warnings', + insights: (): string => '/insights', insightNew: (filters?: AnyPartialFilterType, dashboardId?: DashboardType['id'] | null, query?: string): string => combineUrl('/insights/new', dashboardId ? { dashboard: dashboardId } : {}, { ...(filters ? { filters } : {}), diff --git a/frontend/src/scenes/userLogic.ts b/frontend/src/scenes/userLogic.ts index 7bc8275f8594c..6baff50a4c1ec 100644 --- a/frontend/src/scenes/userLogic.ts +++ b/frontend/src/scenes/userLogic.ts @@ -4,11 +4,11 @@ import { loaders } from 'kea-loaders' import { urlToAction } from 'kea-router' import api from 'lib/api' import { DashboardCompatibleScenes } from 'lib/components/SceneDashboardChoice/sceneDashboardChoiceModalLogic' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { getAppContext } from 'lib/utils/getAppContext' import posthog from 'posthog-js' -import { AvailableFeature, OrganizationBasicType, ProductKey, UserType } from '~/types' +import { AvailableFeature, OrganizationBasicType, ProductKey, UserTheme, UserType } from '~/types' import type { userLogicType } from './userLogicType' @@ -232,6 +232,13 @@ export const userLogic = kea([ ) || [] : [], ], + + themeMode: [ + (s) => [s.user], + (user): UserTheme => { + return user?.theme_mode || 'light' + }, + ], }), afterMount(({ actions }) => { const preloadedUser = getAppContext()?.current_user diff --git a/frontend/src/scenes/web-analytics/WebAnalyticsHealthCheck.tsx b/frontend/src/scenes/web-analytics/WebAnalyticsHealthCheck.tsx index 34cad0e5dc31a..85c9c9b57585d 100644 --- a/frontend/src/scenes/web-analytics/WebAnalyticsHealthCheck.tsx +++ b/frontend/src/scenes/web-analytics/WebAnalyticsHealthCheck.tsx @@ -11,12 +11,12 @@ export const WebAnalyticsHealthCheck = (): JSX.Element | null => { return null } - if (statusCheck.shouldWarnAboutNoPageviews) { + if (!statusCheck.isSendingPageViews) { return (

No $pageview{' '} - {statusCheck.shouldWarnAboutNoPageleaves ? ( + {!statusCheck.isSendingPageLeaves ? ( <> or $pageleave{' '} @@ -30,7 +30,7 @@ export const WebAnalyticsHealthCheck = (): JSX.Element | null => {

) - } else if (statusCheck.shouldWarnAboutNoPageleaves) { + } else if (!statusCheck.isSendingPageLeaves) { return (

diff --git a/frontend/src/scenes/web-analytics/WebAnalyticsTile.tsx b/frontend/src/scenes/web-analytics/WebAnalyticsTile.tsx index 74a7e8b156efd..f2ffbba78cd66 100644 --- a/frontend/src/scenes/web-analytics/WebAnalyticsTile.tsx +++ b/frontend/src/scenes/web-analytics/WebAnalyticsTile.tsx @@ -174,6 +174,16 @@ export const webAnalyticsDataTableQueryContext: QueryContext = { render: NumericCell, align: 'right', }, + average_scroll_percentage: { + title: 'Average Scroll', + render: PercentageCell, + align: 'right', + }, + scroll_gt80_percentage: { + title: 'Deep Scroll Rate', + render: PercentageCell, + align: 'right', + }, }, } @@ -184,7 +194,7 @@ export const WebStatsTrendTile = ({ query: InsightVizNode showIntervalTile?: boolean }): JSX.Element => { - const { togglePropertyFilter, setGeographyTab, setDeviceTab, setInterval } = useActions(webAnalyticsLogic) + const { togglePropertyFilter, setInterval } = useActions(webAnalyticsLogic) const { hasCountryFilter, deviceTab, @@ -201,11 +211,9 @@ export const WebStatsTrendTile = ({ if (!worldMapPropertyName) { return } - togglePropertyFilter(PropertyFilterType.Event, worldMapPropertyName, breakdownValue) - if (!hasCountryFilter) { - // if we just added a country filter, switch to the region tab, as the world map will not be useful - setGeographyTab(GeographyTab.REGIONS) - } + togglePropertyFilter(PropertyFilterType.Event, worldMapPropertyName, breakdownValue, { + geographyTab: hasCountryFilter ? undefined : GeographyTab.REGIONS, + }) }, [togglePropertyFilter, worldMapPropertyName] ) @@ -226,16 +234,20 @@ export const WebStatsTrendTile = ({ if (!deviceTypePropertyName) { return } - togglePropertyFilter(PropertyFilterType.Event, deviceTypePropertyName, breakdownValue) // switch to a different tab if we can, try them in this order: DeviceType Browser OS + let newTab: DeviceTab | undefined = undefined if (deviceTab !== DeviceTab.DEVICE_TYPE && !hasDeviceTypeFilter) { - setDeviceTab(DeviceTab.DEVICE_TYPE) + newTab = DeviceTab.DEVICE_TYPE } else if (deviceTab !== DeviceTab.BROWSER && !hasBrowserFilter) { - setDeviceTab(DeviceTab.BROWSER) + newTab = DeviceTab.BROWSER } else if (deviceTab !== DeviceTab.OS && !hasOSFilter) { - setDeviceTab(DeviceTab.OS) + newTab = DeviceTab.OS } + + togglePropertyFilter(PropertyFilterType.Event, deviceTypePropertyName, breakdownValue, { + deviceTab: newTab, + }) }, [togglePropertyFilter, deviceTypePropertyName, deviceTab, hasDeviceTypeFilter, hasBrowserFilter, hasOSFilter] ) diff --git a/frontend/src/scenes/web-analytics/WebDashboard.tsx b/frontend/src/scenes/web-analytics/WebDashboard.tsx index bade0035640b4..b2e48e71b1213 100644 --- a/frontend/src/scenes/web-analytics/WebDashboard.tsx +++ b/frontend/src/scenes/web-analytics/WebDashboard.tsx @@ -94,7 +94,7 @@ const Tiles = (): JSX.Element => { const { tiles } = useValues(webAnalyticsLogic) return ( -

+
{tiles.map((tile, i) => { if ('query' in tile) { const { query, title, layout } = tile @@ -104,6 +104,7 @@ const Tiles = (): JSX.Element => { className={clsx( 'col-span-1 row-span-1 flex flex-col', `md:col-span-${layout.colSpan ?? 6} md:row-span-${layout.rowSpan ?? 1}`, + `xxl:order-${layout.orderLarge ?? 12}`, layout.className )} > @@ -128,7 +129,8 @@ const TabsTileItem = ({ tile }: { tile: TabsTile }): JSX.Element => { ({ value: id, label: linkText }))} /> ) : ( - ({ key: id, label: linkText }))} + ({ label: linkText, value: id }))} + onChange={(value) => setActiveTabId(value)} + value={activeTabId} /> )}
diff --git a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts index 66048d2fb49f5..f3408a2fd534c 100644 --- a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts +++ b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts @@ -1,5 +1,6 @@ import { actions, afterMount, connect, kea, path, reducers, selectors } from 'kea' import { loaders } from 'kea-loaders' +import { actionToUrl, urlToAction } from 'kea-router' import { windowValues } from 'kea-window-values' import api from 'lib/api' import { RETENTION_FIRST_TIME, STALE_EVENT_SECONDS } from 'lib/constants' @@ -29,9 +30,10 @@ import { import type { webAnalyticsLogicType } from './webAnalyticsLogicType' export interface WebTileLayout { - colSpan?: number + colSpan?: number | 'full' rowSpan?: number className?: string + orderLarge?: number } interface BaseTile { @@ -92,8 +94,9 @@ export enum GeographyTab { } export interface WebAnalyticsStatusCheck { - shouldWarnAboutNoPageviews: boolean - shouldWarnAboutNoPageleaves: boolean + isSendingPageViews: boolean + isSendingPageLeaves: boolean + isSendingPageLeavesScroll: boolean } export const GEOIP_PLUGIN_URLS = [ @@ -114,11 +117,19 @@ export const webAnalyticsLogic = kea([ togglePropertyFilter: ( type: PropertyFilterType.Event | PropertyFilterType.Person, key: string, - value: string | number + value: string | number, + tabChange?: { + graphsTab?: string + sourceTab?: string + deviceTab?: string + pathTab?: string + geographyTab?: string + } ) => ({ type, key, value, + tabChange, }), setGraphsTab: (tab: string) => ({ tab, @@ -135,6 +146,19 @@ export const webAnalyticsLogic = kea([ setGeographyTab: (tab: string) => ({ tab }), setDates: (dateFrom: string | null, dateTo: string | null) => ({ dateFrom, dateTo }), setInterval: (interval: IntervalType) => ({ interval }), + setStateFromUrl: (state: { + filters: WebAnalyticsPropertyFilters + dateFrom: string | null + dateTo: string | null + interval: IntervalType | null + graphsTab: string | null + sourceTab: string | null + deviceTab: string | null + pathTab: string | null + geographyTab: string | null + }) => ({ + state, + }), }), reducers({ webAnalyticsFilters: [ @@ -184,36 +208,47 @@ export const webAnalyticsLogic = kea([ return [...oldPropertyFilters, newFilter] } }, + setStateFromUrl: (_, { state }) => state.filters, }, ], - graphsTab: [ - GraphsTab.UNIQUE_USERS as string, + _graphsTab: [ + null as string | null, { setGraphsTab: (_, { tab }) => tab, + setStateFromUrl: (_, { state }) => state.graphsTab, + togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.graphsTab || oldTab, }, ], - sourceTab: [ - SourceTab.REFERRING_DOMAIN as string, + _sourceTab: [ + null as string | null, { setSourceTab: (_, { tab }) => tab, + setStateFromUrl: (_, { state }) => state.sourceTab, + togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.sourceTab || oldTab, }, ], - deviceTab: [ - DeviceTab.DEVICE_TYPE as string, + _deviceTab: [ + null as string | null, { setDeviceTab: (_, { tab }) => tab, + setStateFromUrl: (_, { state }) => state.deviceTab, + togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.deviceTab || oldTab, }, ], - pathTab: [ - PathTab.PATH as string, + _pathTab: [ + null as string | null, { setPathTab: (_, { tab }) => tab, + setStateFromUrl: (_, { state }) => state.pathTab, + togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.pathTab || oldTab, }, ], - geographyTab: [ - GeographyTab.MAP as string, + _geographyTab: [ + null as string | null, { setGeographyTab: (_, { tab }) => tab, + setStateFromUrl: (_, { state }) => state.geographyTab, + togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.geographyTab || oldTab, }, ], dateFilter: [ @@ -236,30 +271,48 @@ export const webAnalyticsLogic = kea([ interval, } }, + setStateFromUrl: (_, { state: { dateTo, dateFrom, interval } }) => { + if (!dateFrom && !dateTo) { + dateFrom = initialDateFrom + dateTo = initialDateTo + } + return { + dateTo, + dateFrom, + interval: interval || getDefaultInterval(dateFrom, dateTo), + } + }, }, ], }), selectors(({ actions, values }) => ({ + graphsTab: [(s) => [s._graphsTab], (graphsTab: string | null) => graphsTab || GraphsTab.UNIQUE_USERS], + sourceTab: [(s) => [s._sourceTab], (sourceTab: string | null) => sourceTab || SourceTab.REFERRING_DOMAIN], + deviceTab: [(s) => [s._deviceTab], (deviceTab: string | null) => deviceTab || DeviceTab.DEVICE_TYPE], + pathTab: [(s) => [s._pathTab], (pathTab: string | null) => pathTab || PathTab.PATH], + geographyTab: [(s) => [s._geographyTab], (geographyTab: string | null) => geographyTab || GeographyTab.MAP], tiles: [ (s) => [ s.webAnalyticsFilters, s.graphsTab, - s.pathTab, - s.deviceTab, s.sourceTab, + s.deviceTab, + s.pathTab, s.geographyTab, s.dateFilter, + () => values.statusCheck, () => values.isGreaterThanMd, () => values.shouldShowGeographyTile, ], ( webAnalyticsFilters, graphsTab, - pathTab, - deviceTab, sourceTab, + deviceTab, + pathTab, geographyTab, { dateFrom, dateTo, interval }, + statusCheck, isGreaterThanMd: boolean, shouldShowGeographyTile ): WebDashboardTile[] => { @@ -272,7 +325,8 @@ export const webAnalyticsLogic = kea([ const tiles: (WebDashboardTile | null)[] = [ { layout: { - colSpan: 12, + colSpan: 'full', + orderLarge: 0, }, query: { kind: NodeKind.WebOverviewQuery, @@ -282,7 +336,8 @@ export const webAnalyticsLogic = kea([ }, { layout: { - colSpan: 6, + colSpan: 2, + orderLarge: 1, }, activeTabId: graphsTab, setTabId: actions.setGraphsTab, @@ -382,7 +437,8 @@ export const webAnalyticsLogic = kea([ }, { layout: { - colSpan: 6, + colSpan: 2, + orderLarge: 4, }, activeTabId: pathTab, setTabId: actions.setPathTab, @@ -399,6 +455,7 @@ export const webAnalyticsLogic = kea([ properties: webAnalyticsFilters, breakdownBy: WebStatsBreakdown.Page, dateRange, + includeScrollDepth: statusCheck?.isSendingPageLeavesScroll, }, embedded: false, }, @@ -415,6 +472,7 @@ export const webAnalyticsLogic = kea([ properties: webAnalyticsFilters, breakdownBy: WebStatsBreakdown.InitialPage, dateRange, + includeScrollDepth: statusCheck?.isSendingPageLeavesScroll, }, embedded: false, }, @@ -423,7 +481,8 @@ export const webAnalyticsLogic = kea([ }, { layout: { - colSpan: 6, + colSpan: 1, + orderLarge: 2, }, activeTabId: sourceTab, setTabId: actions.setSourceTab, @@ -537,15 +596,16 @@ export const webAnalyticsLogic = kea([ }, { layout: { - colSpan: 6, + colSpan: 1, + orderLarge: 3, }, activeTabId: deviceTab, setTabId: actions.setDeviceTab, tabs: [ { id: DeviceTab.DEVICE_TYPE, - title: 'Top Device Types', - linkText: 'Device Type', + title: 'Device types', + linkText: 'Device type', query: { kind: NodeKind.InsightVizNode, source: { @@ -610,41 +670,13 @@ export const webAnalyticsLogic = kea([ }, ], }, - { - title: 'Retention', - layout: { - colSpan: 12, - }, - query: { - kind: NodeKind.InsightVizNode, - source: { - kind: NodeKind.RetentionQuery, - properties: webAnalyticsFilters, - dateRange, - filterTestAccounts: true, - retentionFilter: { - retention_type: RETENTION_FIRST_TIME, - retention_reference: 'total', - total_intervals: isGreaterThanMd ? 8 : 5, - period: RetentionPeriod.Week, - }, - }, - vizSpecificOptions: { - [InsightType.RETENTION]: { - hideLineGraph: true, - hideSizeColumn: !isGreaterThanMd, - useSmallLayout: !isGreaterThanMd, - }, - }, - embedded: true, - }, - }, + shouldShowGeographyTile ? { layout: { - colSpan: 12, + colSpan: 'full', }, - activeTabId: geographyTab, + activeTabId: geographyTab || GeographyTab.MAP, setTabId: actions.setGeographyTab, tabs: [ { @@ -725,6 +757,35 @@ export const webAnalyticsLogic = kea([ ], } : null, + { + title: 'Retention', + layout: { + colSpan: 2, + }, + query: { + kind: NodeKind.InsightVizNode, + source: { + kind: NodeKind.RetentionQuery, + properties: webAnalyticsFilters, + dateRange, + filterTestAccounts: true, + retentionFilter: { + retention_type: RETENTION_FIRST_TIME, + retention_reference: 'total', + total_intervals: isGreaterThanMd ? 8 : 5, + period: RetentionPeriod.Week, + }, + }, + vizSpecificOptions: { + [InsightType.RETENTION]: { + hideLineGraph: true, + hideSizeColumn: !isGreaterThanMd, + useSmallLayout: !isGreaterThanMd, + }, + }, + embedded: true, + }, + }, ] return tiles.filter(isNotNil) }, @@ -760,7 +821,7 @@ export const webAnalyticsLogic = kea([ statusCheck: { __default: null as WebAnalyticsStatusCheck | null, loadStatusCheck: async (): Promise => { - const [pageviewResult, pageleaveResult] = await Promise.allSettled([ + const [pageviewResult, pageleaveResult, pageleaveScroll] = await Promise.allSettled([ api.eventDefinitions.list({ event_type: EventDefinitionType.Event, search: '$pageview', @@ -769,6 +830,10 @@ export const webAnalyticsLogic = kea([ event_type: EventDefinitionType.Event, search: '$pageleave', }), + api.propertyDefinitions.list({ + event_names: ['$pageleave'], + properties: ['$prev_pageview_max_content_percentage'], + }), ]) // no need to worry about pagination here, event names beginning with $ are reserved, and we're not @@ -783,12 +848,19 @@ export const webAnalyticsLogic = kea([ ? pageleaveResult.value.results.find((r) => r.name === '$pageleave') : undefined - const shouldWarnAboutNoPageviews = !pageviewEntry || isDefinitionStale(pageviewEntry) - const shouldWarnAboutNoPageleaves = !pageleaveEntry || isDefinitionStale(pageleaveEntry) + const pageleaveScrollEntry = + pageleaveScroll.status === 'fulfilled' + ? pageleaveScroll.value.results.find((r) => r.name === '$prev_pageview_max_content_percentage') + : undefined + + const isSendingPageViews = !!pageviewEntry && !isDefinitionStale(pageviewEntry) + const isSendingPageLeaves = !!pageleaveEntry && !isDefinitionStale(pageleaveEntry) + const isSendingPageLeavesScroll = !!pageleaveScrollEntry && !isDefinitionStale(pageleaveScrollEntry) return { - shouldWarnAboutNoPageviews, - shouldWarnAboutNoPageleaves, + isSendingPageViews, + isSendingPageLeaves, + isSendingPageLeavesScroll, } }, }, @@ -837,6 +909,77 @@ export const webAnalyticsLogic = kea([ windowValues({ isGreaterThanMd: (window: Window) => window.innerWidth > 768, }), + + actionToUrl(({ values }) => { + const stateToUrl = (): string => { + const { + webAnalyticsFilters, + dateFilter: { dateTo, dateFrom, interval }, + sourceTab, + deviceTab, + pathTab, + geographyTab, + graphsTab, + } = values + + const urlParams = new URLSearchParams() + if (webAnalyticsFilters.length > 0) { + urlParams.set('filters', JSON.stringify(webAnalyticsFilters)) + } + if (dateFrom !== initialDateFrom || dateTo !== initialDateTo || interval !== initialInterval) { + urlParams.set('date_from', dateFrom ?? '') + urlParams.set('date_to', dateTo ?? '') + urlParams.set('interval', interval ?? '') + } + if (deviceTab) { + urlParams.set('device_tab', deviceTab) + } + if (sourceTab) { + urlParams.set('source_tab', sourceTab) + } + if (graphsTab) { + urlParams.set('graphs_tab', graphsTab) + } + if (pathTab) { + urlParams.set('path_tab', pathTab) + } + if (geographyTab) { + urlParams.set('geography_tab', geographyTab) + } + return `/web?${urlParams.toString()}` + } + + return { + setWebAnalyticsFilters: stateToUrl, + togglePropertyFilter: stateToUrl, + setDates: stateToUrl, + setInterval: stateToUrl, + setDeviceTab: stateToUrl, + setSourceTab: stateToUrl, + setGraphsTab: stateToUrl, + setPathTab: stateToUrl, + setGeographyTab: stateToUrl, + } + }), + + urlToAction(({ actions }) => ({ + '/web': ( + _, + { filters, date_from, date_to, interval, device_tab, source_tab, graphs_tab, path_tab, geography_tab } + ) => { + actions.setStateFromUrl({ + filters: filters || initialWebAnalyticsFilter, + dateFrom: date_from || null, + dateTo: date_to || null, + interval: interval || null, + deviceTab: device_tab || null, + sourceTab: source_tab || null, + graphsTab: graphs_tab || null, + pathTab: path_tab || null, + geographyTab: geography_tab || null, + }) + }, + })), ]) const isDefinitionStale = (definition: EventDefinition | PropertyDefinition): boolean => { diff --git a/frontend/src/stories/How to build a form.stories.mdx b/frontend/src/stories/How to build a form.stories.mdx index 37b69133ab6eb..504cd7714e08c 100644 --- a/frontend/src/stories/How to build a form.stories.mdx +++ b/frontend/src/stories/How to build a form.stories.mdx @@ -163,7 +163,7 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element { )} - + ) } diff --git a/frontend/src/styles/global.scss b/frontend/src/styles/global.scss index e1aae6b3218c5..45c21e1e367b3 100644 --- a/frontend/src/styles/global.scss +++ b/frontend/src/styles/global.scss @@ -467,6 +467,10 @@ input::-ms-clear { &.main-app-content--container { align-self: center; max-width: 72rem; + + @include screen($xxl) { + max-width: 108rem; + } } @include screen($sm) { diff --git a/frontend/src/styles/utilities.scss b/frontend/src/styles/utilities.scss index 705c3a4942589..04d595ea54910 100644 --- a/frontend/src/styles/utilities.scss +++ b/frontend/src/styles/utilities.scss @@ -854,6 +854,13 @@ $decorations: underline, overline, line-through, no-underline; row-gap: #{$space * 0.25}rem; } } + + // Order + @for $i from 1 through 12 { + .#{$prefix}order-#{$i} { + order: #{$i}; + } + } } } @@ -1140,6 +1147,10 @@ $decorations: underline, overline, line-through, no-underline; font-family: var(--font-mono); } +.ligatures-none { + font-variant-ligatures: none; +} + .opacity-0 { opacity: 0; } diff --git a/frontend/src/styles/vars.scss b/frontend/src/styles/vars.scss index 096baebf9b890..98a7d1c711eef 100644 --- a/frontend/src/styles/vars.scss +++ b/frontend/src/styles/vars.scss @@ -15,7 +15,7 @@ $screens: ( 'xxl': $xxl, ); $tiny_spaces: 0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20; -$humongous_spaces: 24, 30, 32, 40, 50, 60, 80, 100, 120, 140, 160, 180, 200; +$humongous_spaces: 24, 30, 32, 40, 50, 60, 70, 80, 100, 120, 140, 160, 180, 200; $all_spaces: list.join($tiny_spaces, $humongous_spaces); $flex_sizes: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10; $leadings: 3, 4, 5, 6, 7, 8, 9, 10; diff --git a/frontend/src/toolbar/actions/actionsTabLogic.tsx b/frontend/src/toolbar/actions/actionsTabLogic.tsx index e2cf38bfed1dd..4c37a5d10d1ec 100644 --- a/frontend/src/toolbar/actions/actionsTabLogic.tsx +++ b/frontend/src/toolbar/actions/actionsTabLogic.tsx @@ -2,7 +2,7 @@ import { actions, connect, kea, listeners, path, reducers, selectors } from 'kea import { forms } from 'kea-forms' import { subscriptions } from 'kea-subscriptions' import api from 'lib/api' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { urls } from 'scenes/urls' import { actionsLogic } from '~/toolbar/actions/actionsLogic' diff --git a/frontend/src/toolbar/toolbarConfigLogic.ts b/frontend/src/toolbar/toolbarConfigLogic.ts index e45a2c7b8ee1b..1b4638b8f39f8 100644 --- a/frontend/src/toolbar/toolbarConfigLogic.ts +++ b/frontend/src/toolbar/toolbarConfigLogic.ts @@ -1,6 +1,6 @@ import { actions, afterMount, kea, listeners, path, props, reducers, selectors } from 'kea' import { combineUrl, encodeParams } from 'kea-router' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { posthog } from '~/toolbar/posthog' import { ToolbarProps } from '~/types' diff --git a/frontend/src/toolbar/toolbarLogic.ts b/frontend/src/toolbar/toolbarLogic.ts index 1394e1d1d05f2..5718833bf2b12 100644 --- a/frontend/src/toolbar/toolbarLogic.ts +++ b/frontend/src/toolbar/toolbarLogic.ts @@ -1,5 +1,5 @@ import { actions, afterMount, kea, listeners, path, props, reducers, selectors } from 'kea' -import { lemonToast } from 'lib/lemon-ui/lemonToast' +import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { actionsTabLogic } from '~/toolbar/actions/actionsTabLogic' import { toolbarButtonLogic } from '~/toolbar/button/toolbarButtonLogic' diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 9e405a1e2a471..dda2cbdeb9817 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -150,6 +150,7 @@ interface UserBaseType { uuid: string distinct_id: string first_name: string + last_name?: string email: string } @@ -168,6 +169,8 @@ export interface SceneDashboardChoice { dashboard: number | DashboardBasicType } +export type UserTheme = 'light' | 'dark' | 'system' + /** Full User model. */ export interface UserType extends UserBaseType { date_joined: string @@ -189,8 +192,7 @@ export interface UserType extends UserBaseType { has_social_auth: boolean has_seen_product_intro_for?: Record scene_personalisation?: SceneDashboardChoice[] - /** Null means "sync with system". */ - theme_mode: 'light' | 'dark' | null + theme_mode?: UserTheme | null } export interface NotificationSettings { @@ -874,6 +876,7 @@ export interface MatchedRecording { interface CommonActorType { id: string | number properties: Record + /** @format date-time */ created_at: string matched_recordings: MatchedRecording[] value_at_data_point: number | null @@ -1762,6 +1765,7 @@ export interface FilterType { breakdown_normalize_url?: boolean breakdowns?: Breakdown[] breakdown_group_type_index?: number | null + breakdown_hide_other_aggregation?: boolean | null aggregation_group_type_index?: number // Groups aggregation } @@ -1881,8 +1885,10 @@ export interface RetentionEntity { export interface RetentionFilterType extends FilterType { retention_type?: RetentionType retention_reference?: 'total' | 'previous' // retention wrt cohort size or previous period - /** @asType integer */ - total_intervals?: number // retention total intervals + /** + * @asType integer + */ + total_intervals?: number returning_entity?: RetentionEntity target_entity?: RetentionEntity period?: RetentionPeriod diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 6ac9f5e85697a..9deccea94b408 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0015_add_verified_properties otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0376_externaldataschema_last_synced_at +posthog: 0379_alter_scheduledchange sessions: 0001_initial social_django: 0010_uid_db_index -two_factor: 0007_auto_20201201_1019 \ No newline at end of file +two_factor: 0007_auto_20201201_1019 diff --git a/package.json b/package.json index 0d5ea062a1328..72857e7ccc199 100644 --- a/package.json +++ b/package.json @@ -142,7 +142,7 @@ "papaparse": "^5.4.1", "pmtiles": "^2.11.0", "posthog-js": "1.96.0", - "posthog-js-lite": "2.0.0-alpha5", + "posthog-js-lite": "2.5.0", "prettier": "^2.8.8", "prop-types": "^15.7.2", "protomaps-themes-base": "2.0.0-alpha.1", diff --git a/plugin-server/src/config/config.ts b/plugin-server/src/config/config.ts index 40afd2c248481..973b9c99eba46 100644 --- a/plugin-server/src/config/config.ts +++ b/plugin-server/src/config/config.ts @@ -129,6 +129,7 @@ export function getDefaultConfig(): PluginsServerConfig { DROP_EVENTS_BY_TOKEN_DISTINCT_ID: '', DROP_EVENTS_BY_TOKEN: '', POE_DEFERRED_WRITES_ENABLED: false, + POE_DEFERRED_WRITES_USE_FLAT_OVERRIDES: false, POE_EMBRACE_JOIN_FOR_TEAMS: '', RELOAD_PLUGIN_JITTER_MAX_MS: 60000, RUSTY_HOOK_FOR_TEAMS: '', diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index f59d196e93d24..5b94900e807f9 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -21,7 +21,11 @@ import { status } from '../utils/status' import { delay } from '../utils/utils' import { AppMetrics } from '../worker/ingestion/app-metrics' import { OrganizationManager } from '../worker/ingestion/organization-manager' -import { DeferredPersonOverrideWorker } from '../worker/ingestion/person-state' +import { + DeferredPersonOverrideWorker, + FlatPersonOverrideWriter, + PersonOverrideWriter, +} from '../worker/ingestion/person-state' import { TeamManager } from '../worker/ingestion/team-manager' import Piscina, { makePiscina as defaultMakePiscina } from '../worker/piscina' import { GraphileWorker } from './graphile-worker/graphile-worker' @@ -437,7 +441,13 @@ export async function startPluginsServer( const postgres = hub?.postgres ?? new PostgresRouter(serverConfig) const kafkaProducer = hub?.kafkaProducer ?? (await createKafkaProducerWrapper(serverConfig)) - personOverridesPeriodicTask = new DeferredPersonOverrideWorker(postgres, kafkaProducer).runTask(5000) + personOverridesPeriodicTask = new DeferredPersonOverrideWorker( + postgres, + kafkaProducer, + serverConfig.POE_DEFERRED_WRITES_USE_FLAT_OVERRIDES + ? new FlatPersonOverrideWriter(postgres) + : new PersonOverrideWriter(postgres) + ).runTask(5000) personOverridesPeriodicTask.promise.catch(async () => { status.error('⚠️', 'Person override worker task crashed! Requesting shutdown...') await closeJobs() diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index 062735c2525b2..0d15899c84aa2 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -200,6 +200,7 @@ export interface PluginsServerConfig { DROP_EVENTS_BY_TOKEN: string POE_EMBRACE_JOIN_FOR_TEAMS: string POE_DEFERRED_WRITES_ENABLED: boolean + POE_DEFERRED_WRITES_USE_FLAT_OVERRIDES: boolean RELOAD_PLUGIN_JITTER_MAX_MS: number RUSTY_HOOK_FOR_TEAMS: string RUSTY_HOOK_URL: string diff --git a/plugin-server/src/worker/ingestion/person-state.ts b/plugin-server/src/worker/ingestion/person-state.ts index df9377a7df425..f49ee1ad334e9 100644 --- a/plugin-server/src/worker/ingestion/person-state.ts +++ b/plugin-server/src/worker/ingestion/person-state.ts @@ -706,13 +706,142 @@ export class PersonOverrideWriter { return id } + + public async getPersonOverrides(teamId: number): Promise { + const { rows } = await this.postgres.query( + PostgresUse.COMMON_WRITE, + SQL` + SELECT + override.team_id, + old_person.uuid as old_person_id, + override_person.uuid as override_person_id, + oldest_event + FROM posthog_personoverride override + LEFT OUTER JOIN posthog_personoverridemapping old_person + ON override.team_id = old_person.team_id AND override.old_person_id = old_person.id + LEFT OUTER JOIN posthog_personoverridemapping override_person + ON override.team_id = override_person.team_id AND override.override_person_id = override_person.id + WHERE override.team_id = ${teamId} + `, + undefined, + 'getPersonOverrides' + ) + return rows.map((row) => ({ + ...row, + oldest_event: DateTime.fromISO(row.oldest_event), + })) + } +} + +export class FlatPersonOverrideWriter { + constructor(private postgres: PostgresRouter) {} + + public async addPersonOverride( + tx: TransactionClient, + overrideDetails: PersonOverrideDetails + ): Promise { + const mergedAt = DateTime.now() + + await this.postgres.query( + tx, + SQL` + INSERT INTO posthog_flatpersonoverride ( + team_id, + old_person_id, + override_person_id, + oldest_event, + version + ) VALUES ( + ${overrideDetails.team_id}, + ${overrideDetails.old_person_id}, + ${overrideDetails.override_person_id}, + ${overrideDetails.oldest_event}, + 0 + ) + `, + undefined, + 'personOverride' + ) + + const { rows: transitiveUpdates } = await this.postgres.query( + tx, + SQL` + UPDATE + posthog_flatpersonoverride + SET + override_person_id = ${overrideDetails.override_person_id}, + version = COALESCE(version, 0)::numeric + 1 + WHERE + team_id = ${overrideDetails.team_id} AND override_person_id = ${overrideDetails.old_person_id} + RETURNING + old_person_id, + version, + oldest_event + `, + undefined, + 'transitivePersonOverrides' + ) + + status.debug('🔁', 'person_overrides_updated', { transitiveUpdates }) + + const personOverrideMessages: ProducerRecord[] = [ + { + topic: KAFKA_PERSON_OVERRIDE, + messages: [ + { + value: JSON.stringify({ + team_id: overrideDetails.team_id, + old_person_id: overrideDetails.old_person_id, + override_person_id: overrideDetails.override_person_id, + oldest_event: castTimestampOrNow(overrideDetails.oldest_event, TimestampFormat.ClickHouse), + merged_at: castTimestampOrNow(mergedAt, TimestampFormat.ClickHouse), + version: 0, + }), + }, + ...transitiveUpdates.map(({ old_person_id, version, oldest_event }) => ({ + value: JSON.stringify({ + team_id: overrideDetails.team_id, + old_person_id: old_person_id, + override_person_id: overrideDetails.override_person_id, + oldest_event: castTimestampOrNow(oldest_event, TimestampFormat.ClickHouse), + merged_at: castTimestampOrNow(mergedAt, TimestampFormat.ClickHouse), + version: version, + }), + })), + ], + }, + ] + + return personOverrideMessages + } + + public async getPersonOverrides(teamId: number): Promise { + const { rows } = await this.postgres.query( + PostgresUse.COMMON_WRITE, + SQL` + SELECT + team_id, + old_person_id, + override_person_id, + oldest_event + FROM posthog_flatpersonoverride + WHERE team_id = ${teamId} + `, + undefined, + 'getPersonOverrides' + ) + return rows.map((row) => ({ + ...row, + team_id: parseInt(row.team_id), // XXX: pg returns bigint as str (reasonably so) + oldest_event: DateTime.fromISO(row.oldest_event), + })) + } } const deferredPersonOverridesWrittenCounter = new Counter({ name: 'deferred_person_overrides_written', help: 'Number of person overrides that have been written as pending', }) - export class DeferredPersonOverrideWriter { constructor(private postgres: PostgresRouter) {} @@ -759,11 +888,11 @@ export class DeferredPersonOverrideWorker { // it just needs to be consistent across all processes. public readonly lockId = 567 - private writer: PersonOverrideWriter - - constructor(private postgres: PostgresRouter, private kafkaProducer: KafkaProducerWrapper) { - this.writer = new PersonOverrideWriter(this.postgres) - } + constructor( + private postgres: PostgresRouter, + private kafkaProducer: KafkaProducerWrapper, + private writer: PersonOverrideWriter | FlatPersonOverrideWriter + ) {} /** * Process all (or up to the given limit) pending overrides. diff --git a/plugin-server/tests/worker/ingestion/person-state.test.ts b/plugin-server/tests/worker/ingestion/person-state.test.ts index 492beab70ca0d..318c3504d3e98 100644 --- a/plugin-server/tests/worker/ingestion/person-state.test.ts +++ b/plugin-server/tests/worker/ingestion/person-state.test.ts @@ -11,6 +11,7 @@ import { UUIDT } from '../../../src/utils/utils' import { DeferredPersonOverrideWorker, DeferredPersonOverrideWriter, + FlatPersonOverrideWriter, PersonOverrideWriter, PersonState, } from '../../../src/worker/ingestion/person-state' @@ -24,57 +25,64 @@ const timestamp = DateTime.fromISO('2020-01-01T12:00:05.200Z').toUTC() const timestamp2 = DateTime.fromISO('2020-02-02T12:00:05.200Z').toUTC() const timestampch = '2020-01-01 12:00:05.000' -async function fetchPostgresPersonIdOverrides(hub: Hub, teamId: number): Promise<[string, string][]> { - const result = await hub.db.postgres.query( - PostgresUse.COMMON_WRITE, - ` - WITH overrides AS ( - SELECT id, old_person_id, override_person_id - FROM posthog_personoverride - WHERE team_id = ${teamId} - ORDER BY id - ) - SELECT - mapping.uuid AS old_person_id, - overrides_mapping.uuid AS override_person_id - FROM - overrides AS first - JOIN - posthog_personoverridemapping AS mapping ON first.old_person_id = mapping.id - JOIN ( - SELECT - second.id AS id, - uuid - FROM - overrides AS second - JOIN posthog_personoverridemapping AS mapping ON second.override_person_id = mapping.id - ) AS overrides_mapping ON overrides_mapping.id = first.id - `, - undefined, - 'fetchPersonIdOverrides' - ) - return result.rows.map(({ old_person_id, override_person_id }) => [old_person_id, override_person_id]).sort() as [ - string, - string - ][] -} - interface PersonOverridesMode { + supportsSyncTransaction: boolean getWriter(hub: Hub): PersonOverrideWriter | DeferredPersonOverrideWriter - fetchPostgresPersonIdOverrides(hub: Hub, teamId: number): Promise<[string, string][]> + fetchPostgresPersonIdOverrides( + hub: Hub, + teamId: number + ): Promise> } const PersonOverridesModes: Record = { disabled: undefined, - immediate: { + 'immediate, with mappings': { + supportsSyncTransaction: true, getWriter: (hub) => new PersonOverrideWriter(hub.db.postgres), - fetchPostgresPersonIdOverrides: (hub, teamId) => fetchPostgresPersonIdOverrides(hub, teamId), + fetchPostgresPersonIdOverrides: async (hub, teamId) => { + const writer = new PersonOverrideWriter(hub.db.postgres) // XXX: ideally would reference ``this``, not new instance + return new Set( + (await writer.getPersonOverrides(teamId)).map(({ old_person_id, override_person_id }) => ({ + old_person_id, + override_person_id, + })) + ) + }, + }, + 'deferred, with mappings': { + supportsSyncTransaction: false, + getWriter: (hub) => new DeferredPersonOverrideWriter(hub.db.postgres), + fetchPostgresPersonIdOverrides: async (hub, teamId) => { + const syncWriter = new PersonOverrideWriter(hub.db.postgres) + await new DeferredPersonOverrideWorker( + hub.db.postgres, + hub.db.kafkaProducer, + syncWriter + ).processPendingOverrides() + return new Set( + (await syncWriter.getPersonOverrides(teamId)).map(({ old_person_id, override_person_id }) => ({ + old_person_id, + override_person_id, + })) + ) + }, }, - deferred: { + 'deferred, without mappings (flat)': { + supportsSyncTransaction: false, getWriter: (hub) => new DeferredPersonOverrideWriter(hub.db.postgres), fetchPostgresPersonIdOverrides: async (hub, teamId) => { - await new DeferredPersonOverrideWorker(hub.db.postgres, hub.db.kafkaProducer).processPendingOverrides() - return await fetchPostgresPersonIdOverrides(hub, teamId) + const syncWriter = new FlatPersonOverrideWriter(hub.db.postgres) + await new DeferredPersonOverrideWorker( + hub.db.postgres, + hub.db.kafkaProducer, + syncWriter + ).processPendingOverrides() + return new Set( + (await syncWriter.getPersonOverrides(teamId)).map(({ old_person_id, override_person_id }) => ({ + old_person_id, + override_person_id, + })) + ) }, }, } @@ -1571,7 +1579,7 @@ describe('PersonState.update()', () => { // verify Postgres person_id overrides, if applicable if (overridesMode) { const overrides = await overridesMode.fetchPostgresPersonIdOverrides(hub, teamId) - expect(overrides).toEqual([[second.uuid, first.uuid]]) + expect(overrides).toEqual(new Set([{ old_person_id: second.uuid, override_person_id: first.uuid }])) // & CH person overrides // TODO } @@ -1733,8 +1741,8 @@ describe('PersonState.update()', () => { }) it(`does not commit partial transactions on override conflicts`, async () => { - if (overridesMode !== PersonOverridesModes.immediate) { - return // this behavior is only supported with immediate overrides + if (!overridesMode?.supportsSyncTransaction) { + return } const first: Person = await hub.db.createPerson( timestamp, @@ -1822,7 +1830,7 @@ describe('PersonState.update()', () => { // verify Postgres person_id overrides const overridesAfterFailure = await overridesMode!.fetchPostgresPersonIdOverrides(hub, teamId) - expect(overridesAfterFailure).toEqual([]) + expect(overridesAfterFailure).toEqual(new Set()) // Now verify we successfully get to our target state if we do not have // any db errors. @@ -1857,7 +1865,7 @@ describe('PersonState.update()', () => { // verify Postgres person_id overrides const overrides = await overridesMode!.fetchPostgresPersonIdOverrides(hub, teamId) - expect(overrides).toEqual([[second.uuid, first.uuid]]) + expect(overrides).toEqual(new Set([{ old_person_id: second.uuid, override_person_id: first.uuid }])) }) it(`handles a chain of overrides being applied concurrently`, async () => { @@ -1998,10 +2006,12 @@ describe('PersonState.update()', () => { // verify Postgres person_id overrides, if applicable if (overridesMode) { const overrides = await overridesMode.fetchPostgresPersonIdOverrides(hub, teamId) - expect(overrides).toEqual([ - [second.uuid, first.uuid], - [third.uuid, first.uuid], - ]) + expect(overrides).toEqual( + new Set([ + { old_person_id: second.uuid, override_person_id: first.uuid }, + { old_person_id: third.uuid, override_person_id: first.uuid }, + ]) + ) } }) @@ -2085,16 +2095,103 @@ describe('PersonState.update()', () => { // verify Postgres person_id overrides, if applicable if (overridesMode) { const overrides = await overridesMode.fetchPostgresPersonIdOverrides(hub, teamId) - expect(overrides).toEqual([ - [second.uuid, first.uuid], - [third.uuid, first.uuid], - ]) + expect(overrides).toEqual( + new Set([ + { old_person_id: second.uuid, override_person_id: first.uuid }, + { old_person_id: third.uuid, override_person_id: first.uuid }, + ]) + ) } }) }) }) }) +const PersonOverridesWriterMode = { + mapping: (hub: Hub) => new PersonOverrideWriter(hub.db.postgres), + flat: (hub: Hub) => new FlatPersonOverrideWriter(hub.db.postgres), +} + +describe.each(Object.keys(PersonOverridesWriterMode))('person overrides writer: %s', (mode) => { + let hub: Hub + let closeHub: () => Promise + + let organizationId: string + let teamId: number + let writer: PersonOverrideWriter | FlatPersonOverrideWriter + + beforeAll(async () => { + ;[hub, closeHub] = await createHub({}) + organizationId = await createOrganization(hub.db.postgres) + writer = PersonOverridesWriterMode[mode](hub) + }) + + beforeEach(async () => { + teamId = await createTeam(hub.db.postgres, organizationId) + }) + + afterAll(async () => { + await closeHub() + }) + + it('handles direct overrides', async () => { + const { postgres } = hub.db + + const defaults = { + team_id: teamId, + oldest_event: DateTime.fromMillis(0), + } + + const override = { + old_person_id: new UUIDT().toString(), + override_person_id: new UUIDT().toString(), + } + + await postgres.transaction(PostgresUse.COMMON_WRITE, '', async (tx) => { + await writer.addPersonOverride(tx, { ...defaults, ...override }) + }) + + expect(await writer.getPersonOverrides(teamId)).toEqual([{ ...defaults, ...override }]) + }) + + it('handles transitive overrides', async () => { + const { postgres } = hub.db + + const defaults = { + team_id: teamId, + oldest_event: DateTime.fromMillis(0), + } + + const overrides = [ + { + old_person_id: new UUIDT().toString(), + override_person_id: new UUIDT().toString(), + }, + ] + + overrides.push({ + old_person_id: overrides[0].override_person_id, + override_person_id: new UUIDT().toString(), + }) + + await postgres.transaction(PostgresUse.COMMON_WRITE, '', async (tx) => { + for (const override of overrides) { + await writer.addPersonOverride(tx, { ...defaults, ...override }) + } + }) + + expect(new Set(await writer.getPersonOverrides(teamId))).toEqual( + new Set( + overrides.map(({ old_person_id }) => ({ + old_person_id, + override_person_id: overrides.at(-1)!.override_person_id, + ...defaults, + })) + ) + ) + }) +}) + describe('deferred person overrides', () => { let hub: Hub let closeHub: () => Promise @@ -2104,13 +2201,15 @@ describe('deferred person overrides', () => { let teamId: number let writer: DeferredPersonOverrideWriter + let syncWriter: PersonOverrideWriter let worker: DeferredPersonOverrideWorker beforeAll(async () => { ;[hub, closeHub] = await createHub({}) organizationId = await createOrganization(hub.db.postgres) writer = new DeferredPersonOverrideWriter(hub.db.postgres) - worker = new DeferredPersonOverrideWorker(hub.db.postgres, hub.db.kafkaProducer) + syncWriter = new PersonOverrideWriter(hub.db.postgres) + worker = new DeferredPersonOverrideWorker(hub.db.postgres, hub.db.kafkaProducer, syncWriter) }) beforeEach(async () => { @@ -2161,9 +2260,12 @@ describe('deferred person overrides', () => { expect(await getPendingPersonOverrides()).toMatchObject([]) - expect(await fetchPostgresPersonIdOverrides(hub, teamId)).toEqual([ - [override.old_person_id, override.override_person_id], - ]) + expect( + (await syncWriter.getPersonOverrides(teamId)).map(({ old_person_id, override_person_id }) => [ + old_person_id, + override_person_id, + ]) + ).toEqual([[override.old_person_id, override.override_person_id]]) const clickhouseOverrides = await waitForExpect(async () => { const { data } = await hub.db.clickhouse.querying( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c9cd08057944f..eae5eb053ab23 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.0' +lockfileVersion: '6.1' settings: autoInstallPeers: true @@ -228,8 +228,8 @@ dependencies: specifier: 1.96.0 version: 1.96.0 posthog-js-lite: - specifier: 2.0.0-alpha5 - version: 2.0.0-alpha5 + specifier: 2.5.0 + version: 2.5.0 prettier: specifier: ^2.8.8 version: 2.8.8 @@ -327,7 +327,7 @@ dependencies: optionalDependencies: fsevents: specifier: ^2.3.2 - version: 2.3.3 + version: 2.3.2 devDependencies: '@babel/core': @@ -12227,7 +12227,6 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] requiresBuild: true - dev: true optional: true /fsevents@2.3.3: @@ -16504,8 +16503,8 @@ packages: source-map-js: 1.0.2 dev: true - /posthog-js-lite@2.0.0-alpha5: - resolution: {integrity: sha512-tlkBdypJuvK/s00n4EiQjwYVfuuZv6vt8BF3g1ooIQa2Gz9Vz80p8q3qsPLZ0V5ErGRy6i3Q4fWC9TDzR7GNRQ==} + /posthog-js-lite@2.5.0: + resolution: {integrity: sha512-Urvlp0Vu9h3td0BVFWt0QXFJDoOZcaAD83XM9d91NKMKTVPZtfU0ysoxstIf5mw/ce9ZfuMgpWPaagrZI4rmSg==} dev: false /posthog-js@1.96.0: diff --git a/posthog/api/authentication.py b/posthog/api/authentication.py index 7da0859f0d0c6..47e0e720cc68d 100644 --- a/posthog/api/authentication.py +++ b/posthog/api/authentication.py @@ -11,6 +11,7 @@ PasswordResetTokenGenerator as DefaultPasswordResetTokenGenerator, ) from django.core.exceptions import ValidationError +from django.core.signing import BadSignature from django.db import transaction from django.http import HttpRequest, HttpResponse, JsonResponse from django.shortcuts import redirect @@ -104,10 +105,15 @@ def _check_if_2fa_required(self, user: User) -> bool: # If user has a valid 2FA cookie, use that instead of showing them the 2FA screen for key, value in self.context["request"].COOKIES.items(): if key.startswith(REMEMBER_COOKIE_PREFIX) and value: - if validate_remember_device_cookie(value, user=user, otp_device_id=device.persistent_id): - user.otp_device = device # type: ignore - device.throttle_reset() - return False + try: + if validate_remember_device_cookie(value, user=user, otp_device_id=device.persistent_id): + user.otp_device = device # type: ignore + device.throttle_reset() + return False + except BadSignature: + # Workaround for signature mismatches due to Django upgrades. + # See https://github.com/PostHog/posthog/issues/19350 + pass return True def create(self, validated_data: Dict[str, str]) -> Any: diff --git a/posthog/api/person.py b/posthog/api/person.py index 6eaca610f482d..287f4c256825c 100644 --- a/posthog/api/person.py +++ b/posthog/api/person.py @@ -869,6 +869,7 @@ def retention(self, request: request.Request) -> response.Response: "result": people, "next": next_url, "missing_persons": raw_count - len(people), + "filters": filter.to_dict(), } ) diff --git a/posthog/api/services/query.py b/posthog/api/services/query.py index 283d1ac7ab9ae..b5ca456ee985e 100644 --- a/posthog/api/services/query.py +++ b/posthog/api/services/query.py @@ -27,7 +27,12 @@ "WebTopPagesQuery", "WebStatsTableQuery", ] -QUERY_WITH_RUNNER_NO_CACHE = ["EventsQuery", "PersonsQuery", "HogQLQuery", "SessionsTimelineQuery"] +QUERY_WITH_RUNNER_NO_CACHE = [ + "HogQLQuery", + "EventsQuery", + "PersonsQuery", + "SessionsTimelineQuery", +] def _unwrap_pydantic(response: Any) -> Dict | List: diff --git a/posthog/api/shared.py b/posthog/api/shared.py index 1a497278b68d0..0ea32d6582330 100644 --- a/posthog/api/shared.py +++ b/posthog/api/shared.py @@ -18,6 +18,7 @@ class Meta: "uuid", "distinct_id", "first_name", + "last_name", "email", "is_email_verified", ] diff --git a/posthog/api/test/__snapshots__/test_insight.ambr b/posthog/api/test/__snapshots__/test_insight.ambr index 28544a877b020..3fe43694ef8c1 100644 --- a/posthog/api/test/__snapshots__/test_insight.ambr +++ b/posthog/api/test/__snapshots__/test_insight.ambr @@ -1,34 +1,32 @@ # name: TestInsight.test_insight_funnels_hogql_breakdown ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT array(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person WHERE team_id = 2 - AND event IN ['user did things', 'user signed up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-08 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - AND (and(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 1)) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event IN ['user did things', 'user signed up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-08 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') + AND (and(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 1)) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestInsight.test_insight_funnels_hogql_breakdown.1 @@ -251,19 +249,17 @@ # name: TestInsight.test_insight_trend_hogql_breakdown ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT if(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 'le%ss', 'more') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-08 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT if(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 'le%ss', 'more') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-08 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestInsight.test_insight_trend_hogql_breakdown.1 @@ -295,7 +291,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(if(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 'le%ss', 'more'), (['more', 'le%ss']), (['more', 'le%ss']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(if(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 'le%ss', 'more'), ''), '$$_posthog_breakdown_null_$$'), (['more', 'le%ss']), (['more', 'le%ss']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview' @@ -314,19 +310,17 @@ # name: TestInsight.test_insight_trend_hogql_breakdown_materialized ' /* user_id:0 request:_snapshot_ */ - SELECT groupArray(value) - FROM - (SELECT if(ifNull(less(toInt64OrNull(nullIf(nullIf(events.mat_int_value, ''), 'null')), 10), 0), 'le%ss', 'more') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-08 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT if(ifNull(less(toInt64OrNull(nullIf(nullIf(events.mat_int_value, ''), 'null')), 10), 0), 'le%ss', 'more') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-08 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestInsight.test_insight_trend_hogql_breakdown_materialized.1 @@ -358,7 +352,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(if(ifNull(less(toInt64OrNull(nullIf(nullIf(events.mat_int_value, ''), 'null')), 10), 0), 'le%ss', 'more'), (['more', 'le%ss']), (['more', 'le%ss']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(if(ifNull(less(toInt64OrNull(nullIf(nullIf(events.mat_int_value, ''), 'null')), 10), 0), 'le%ss', 'more'), ''), '$$_posthog_breakdown_null_$$'), (['more', 'le%ss']), (['more', 'le%ss']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview' diff --git a/posthog/api/test/__snapshots__/test_query.ambr b/posthog/api/test/__snapshots__/test_query.ambr index 38ee1f19a66b2..8d9a9a5c2c4c8 100644 --- a/posthog/api/test/__snapshots__/test_query.ambr +++ b/posthog/api/test/__snapshots__/test_query.ambr @@ -1,8 +1,8 @@ # name: TestQuery.test_event_property_filter ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) @@ -18,8 +18,8 @@ # name: TestQuery.test_event_property_filter.1 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) @@ -35,8 +35,8 @@ # name: TestQuery.test_event_property_filter.2 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) @@ -52,8 +52,8 @@ # name: TestQuery.test_event_property_filter_materialized ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), '')) @@ -69,8 +69,8 @@ # name: TestQuery.test_event_property_filter_materialized.1 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), '')) @@ -86,8 +86,8 @@ # name: TestQuery.test_event_property_filter_materialized.2 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), '')) @@ -103,7 +103,7 @@ # name: TestQuery.test_events_query_all_time_date ' /* user_id:0 request:_snapshot_ */ - SELECT events.event + SELECT events.event AS event FROM events WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2023-01-12 12:14:05.000000', 6, 'UTC'))) ORDER BY events.event ASC @@ -116,7 +116,7 @@ # name: TestQuery.test_events_query_all_time_date.1 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event + SELECT events.event AS event FROM events WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2022-01-12 12:14:00.000000', 6, 'UTC'))) ORDER BY events.event ASC @@ -129,7 +129,7 @@ # name: TestQuery.test_events_query_all_time_date.2 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event + SELECT events.event AS event FROM events WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2022-01-01 00:00:00.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2019-01-12 12:14:00.000000', 6, 'UTC'))) ORDER BY events.event ASC @@ -142,8 +142,8 @@ # name: TestQuery.test_full_hogql_query ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key FROM events WHERE equals(events.team_id, 2) @@ -156,8 +156,8 @@ # name: TestQuery.test_full_hogql_query_materialized ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key FROM events WHERE equals(events.team_id, 2) @@ -184,9 +184,9 @@ # name: TestQuery.test_full_hogql_query_view.1 ' /* user_id:0 request:_snapshot_ */ - SELECT event_view.event, - event_view.distinct_id, - event_view.key + SELECT event_view.event AS event, + event_view.distinct_id AS distinct_id, + event_view.key AS key FROM (SELECT events.event AS event, events.distinct_id AS distinct_id, @@ -202,8 +202,8 @@ # name: TestQuery.test_hogql_property_filter ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) @@ -219,8 +219,8 @@ # name: TestQuery.test_hogql_property_filter.1 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) @@ -236,8 +236,8 @@ # name: TestQuery.test_hogql_property_filter.2 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) @@ -253,8 +253,8 @@ # name: TestQuery.test_hogql_property_filter.3 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) @@ -270,8 +270,8 @@ # name: TestQuery.test_hogql_property_filter_materialized ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), '')) @@ -287,8 +287,8 @@ # name: TestQuery.test_hogql_property_filter_materialized.1 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), '')) @@ -304,8 +304,8 @@ # name: TestQuery.test_hogql_property_filter_materialized.2 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), '')) @@ -321,8 +321,8 @@ # name: TestQuery.test_hogql_property_filter_materialized.3 ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), '')) @@ -338,8 +338,8 @@ # name: TestQuery.test_person_property_filter ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) @@ -352,11 +352,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -372,8 +372,8 @@ # name: TestQuery.test_person_property_filter_materialized ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, + SELECT events.event AS event, + events.distinct_id AS distinct_id, nullIf(nullIf(events.mat_key, ''), 'null') AS key, 'a%sd', concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), '')) @@ -386,11 +386,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, nullIf(nullIf(person.pmat_email, ''), 'null') AS properties___email FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -468,9 +468,9 @@ # name: TestQuery.test_select_event_person ' /* user_id:0 request:_snapshot_ */ - SELECT events.event, - events.distinct_id, - events.distinct_id + SELECT events.event AS event, + events.distinct_id AS distinct_id, + events.distinct_id AS distinct_id FROM events WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC'))) ORDER BY events.event ASC @@ -484,8 +484,8 @@ ' /* user_id:0 request:_snapshot_ */ SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '') AS key, - events.event, - events.distinct_id, + events.event AS event, + events.distinct_id AS distinct_id, concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), '')) FROM events WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC'))) @@ -500,7 +500,7 @@ ' /* user_id:0 request:_snapshot_ */ SELECT tuple(events.uuid, events.event, events.properties, toTimeZone(events.timestamp, 'UTC'), events.team_id, events.distinct_id, events.elements_chain, toTimeZone(events.created_at, 'UTC')), - events.event + events.event AS event FROM events WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC'))) ORDER BY tuple(events.uuid, events.event, events.properties, toTimeZone(events.timestamp, 'UTC'), events.team_id, events.distinct_id, events.elements_chain, toTimeZone(events.created_at, 'UTC')) ASC @@ -514,7 +514,7 @@ ' /* user_id:0 request:_snapshot_ */ SELECT count(), - events.event + events.event AS event FROM events WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC'))) GROUP BY events.event @@ -529,7 +529,7 @@ ' /* user_id:0 request:_snapshot_ */ SELECT count(), - events.event + events.event AS event FROM events WHERE and(equals(events.team_id, 2), or(equals(events.event, 'sign up'), ifNull(like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', ''), '%val2'), 0)), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC'))) GROUP BY events.event diff --git a/posthog/api/test/batch_exports/test_log_entry.py b/posthog/api/test/batch_exports/test_log_entry.py index ad0ee797033b0..b166583ee0b87 100644 --- a/posthog/api/test/batch_exports/test_log_entry.py +++ b/posthog/api/test/batch_exports/test_log_entry.py @@ -38,7 +38,7 @@ def create_batch_export_log_entry( "log_source": "batch_exports", "log_source_id": batch_export_id, "instance_id": run_id, - "timestamp": dt.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f"), + "timestamp": dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f"), "level": level, "message": message, }, @@ -147,7 +147,7 @@ def test_log_level_filter(batch_export, team, level): results = [] timeout = 10 - start = dt.datetime.utcnow() + start = dt.datetime.now(dt.timezone.utc) while not results: results = fetch_batch_export_log_entries( @@ -157,7 +157,7 @@ def test_log_level_filter(batch_export, team, level): after=dt.datetime(2023, 9, 22, 0, 59, 59), before=dt.datetime(2023, 9, 22, 1, 0, 1), ) - if (dt.datetime.utcnow() - start) > dt.timedelta(seconds=timeout): + if (dt.datetime.now(dt.timezone.utc) - start) > dt.timedelta(seconds=timeout): break results.sort(key=lambda record: record.message) @@ -195,7 +195,7 @@ def test_log_level_filter_with_lowercase(batch_export, team, level): results = [] timeout = 10 - start = dt.datetime.utcnow() + start = dt.datetime.now(dt.timezone.utc) while not results: results = fetch_batch_export_log_entries( @@ -205,7 +205,7 @@ def test_log_level_filter_with_lowercase(batch_export, team, level): after=dt.datetime(2023, 9, 22, 0, 59, 59), before=dt.datetime(2023, 9, 22, 1, 0, 1), ) - if (dt.datetime.utcnow() - start) > dt.timedelta(seconds=timeout): + if (dt.datetime.now(dt.timezone.utc) - start) > dt.timedelta(seconds=timeout): break results.sort(key=lambda record: record.message) diff --git a/posthog/api/test/dashboards/test_dashboard_text_tiles.py b/posthog/api/test/dashboards/test_dashboard_text_tiles.py index 509c7b0b9f36a..3bf802f18be6b 100644 --- a/posthog/api/test/dashboards/test_dashboard_text_tiles.py +++ b/posthog/api/test/dashboards/test_dashboard_text_tiles.py @@ -24,6 +24,7 @@ def _serialised_user(user: Optional[User]) -> Optional[Dict[str, Optional[Union[ "distinct_id": user.distinct_id, "email": user.email, "first_name": "", + "last_name": "", "id": user.id, "uuid": str(user.uuid), "is_email_verified": None, diff --git a/posthog/api/test/test_capture.py b/posthog/api/test/test_capture.py index fe16bedeb4f32..886097f71fae9 100644 --- a/posthog/api/test/test_capture.py +++ b/posthog/api/test/test_capture.py @@ -310,7 +310,7 @@ def test_cached_is_randomly_partitioned(self): capacity=1, storage=MemoryStorage(), ) - start = datetime.utcnow() + start = datetime.now(timezone.utc) with patch("posthog.api.capture.LIMITER", new=limiter): with freeze_time(start): diff --git a/posthog/api/test/test_insight.py b/posthog/api/test/test_insight.py index c3ddb7e3b5dfd..5710513597f6b 100644 --- a/posthog/api/test/test_insight.py +++ b/posthog/api/test/test_insight.py @@ -76,6 +76,7 @@ def test_created_updated_and_last_modified(self) -> None: "uuid": str(self.user.uuid), "distinct_id": self.user.distinct_id, "first_name": self.user.first_name, + "last_name": self.user.last_name, "email": self.user.email, "is_email_verified": None, } @@ -84,6 +85,7 @@ def test_created_updated_and_last_modified(self) -> None: "uuid": str(alt_user.uuid), "distinct_id": alt_user.distinct_id, "first_name": alt_user.first_name, + "last_name": alt_user.last_name, "email": alt_user.email, "is_email_verified": None, } diff --git a/posthog/api/test/test_organization_feature_flag.py b/posthog/api/test/test_organization_feature_flag.py index 78e72269b20bb..90576b688aa75 100644 --- a/posthog/api/test/test_organization_feature_flag.py +++ b/posthog/api/test/test_organization_feature_flag.py @@ -50,6 +50,7 @@ def test_get_feature_flag_success(self): "uuid": str(self.user.uuid), "distinct_id": self.user.distinct_id, "first_name": self.user.first_name, + "last_name": self.user.last_name, "email": self.user.email, "is_email_verified": self.user.is_email_verified, }, diff --git a/posthog/api/test/test_organization_invites.py b/posthog/api/test/test_organization_invites.py index 0e52252781963..6d8c9d3a11fc5 100644 --- a/posthog/api/test/test_organization_invites.py +++ b/posthog/api/test/test_organization_invites.py @@ -88,6 +88,7 @@ def test_add_organization_invite_with_email(self, mock_capture): "distinct_id": self.user.distinct_id, "email": self.user.email, "first_name": self.user.first_name, + "last_name": self.user.last_name, "is_email_verified": self.user.is_email_verified, }, "is_expired": False, diff --git a/posthog/api/test/test_organization_members.py b/posthog/api/test/test_organization_members.py index 2416e5552fa9a..132f9a5c4ebe8 100644 --- a/posthog/api/test/test_organization_members.py +++ b/posthog/api/test/test_organization_members.py @@ -97,6 +97,7 @@ def test_change_organization_member_level(self): "uuid": str(user.uuid), "distinct_id": str(user.distinct_id), "first_name": user.first_name, + "last_name": user.last_name, "email": user.email, "is_email_verified": None, }, diff --git a/posthog/api/test/test_query.py b/posthog/api/test/test_query.py index d538e5a241cdf..d8bdb746a67b4 100644 --- a/posthog/api/test/test_query.py +++ b/posthog/api/test/test_query.py @@ -570,7 +570,7 @@ def test_full_hogql_query(self): @patch("posthog.hogql.constants.DEFAULT_RETURNED_ROWS", 10) @patch("posthog.hogql.constants.MAX_SELECT_RETURNED_ROWS", 15) def test_full_hogql_query_limit(self, MAX_SELECT_RETURNED_ROWS=15, DEFAULT_RETURNED_ROWS=10): - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" with freeze_time("2020-01-10 12:00:00"): for _ in range(20): _create_event( @@ -594,7 +594,7 @@ def test_full_hogql_query_limit(self, MAX_SELECT_RETURNED_ROWS=15, DEFAULT_RETUR @patch("posthog.hogql.constants.DEFAULT_RETURNED_ROWS", 10) @patch("posthog.hogql.constants.MAX_SELECT_RETURNED_ROWS", 15) def test_full_hogql_query_limit_exported(self, MAX_SELECT_RETURNED_ROWS=15, DEFAULT_RETURNED_ROWS=10): - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" with freeze_time("2020-01-10 12:00:00"): for _ in range(20): _create_event( @@ -619,7 +619,7 @@ def test_full_hogql_query_limit_exported(self, MAX_SELECT_RETURNED_ROWS=15, DEFA @patch("posthog.hogql.constants.DEFAULT_RETURNED_ROWS", 10) @patch("posthog.hogql.constants.MAX_SELECT_RETURNED_ROWS", 15) def test_full_events_query_limit(self, MAX_SELECT_RETURNED_ROWS=15, DEFAULT_RETURNED_ROWS=10): - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" with freeze_time("2020-01-10 12:00:00"): for _ in range(20): _create_event( @@ -645,7 +645,7 @@ def test_full_events_query_limit(self, MAX_SELECT_RETURNED_ROWS=15, DEFAULT_RETU @patch("posthog.hogql.constants.DEFAULT_RETURNED_ROWS", 10) @patch("posthog.hogql.constants.MAX_SELECT_RETURNED_ROWS", 15) def test_full_events_query_limit_exported(self, MAX_SELECT_RETURNED_ROWS=15, DEFAULT_RETURNED_ROWS=10): - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" with freeze_time("2020-01-10 12:00:00"): for _ in range(20): _create_event( @@ -764,7 +764,7 @@ def test_full_hogql_query_view(self): ) def test_full_hogql_query_values(self): - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" with freeze_time("2020-01-10 12:00:00"): for _ in range(20): _create_event( diff --git a/posthog/api/test/test_scheduled_change.py b/posthog/api/test/test_scheduled_change.py index f7ec88f204c82..415e987cb0216 100644 --- a/posthog/api/test/test_scheduled_change.py +++ b/posthog/api/test/test_scheduled_change.py @@ -13,7 +13,7 @@ def test_can_create_flag_change(self): f"/api/projects/{self.team.id}/scheduled_changes/", data={ "id": 6, - "record_id": 119, + "record_id": "119", "model_name": "FeatureFlag", "payload": payload, "scheduled_at": "2023-12-08T12:00:00Z", @@ -27,6 +27,6 @@ def test_can_create_flag_change(self): assert response.status_code == status.HTTP_201_CREATED, response_data assert ScheduledChange.objects.filter(id=response_data["id"]).exists() assert response_data["model_name"] == "FeatureFlag" - assert response_data["record_id"] == 119 + assert response_data["record_id"] == "119" assert response_data["payload"] == payload assert response_data["created_by"]["id"] == self.user.id diff --git a/posthog/api/test/test_signup.py b/posthog/api/test/test_signup.py index 00c101e4487ee..d4e71415b4569 100644 --- a/posthog/api/test/test_signup.py +++ b/posthog/api/test/test_signup.py @@ -62,6 +62,7 @@ def test_api_sign_up(self, mock_capture): "id": user.pk, "uuid": str(user.uuid), "distinct_id": user.distinct_id, + "last_name": "", "first_name": "John", "email": "hedgehog@posthog.com", "redirect_url": "/", @@ -210,6 +211,7 @@ def test_signup_minimum_attrs(self, mock_capture): "id": user.pk, "uuid": str(user.uuid), "distinct_id": user.distinct_id, + "last_name": "", "first_name": "Jane", "email": "hedgehog2@posthog.com", "redirect_url": "/", @@ -364,6 +366,7 @@ def test_default_dashboard_is_created_on_signup(self): "id": user.pk, "uuid": str(user.uuid), "distinct_id": user.distinct_id, + "last_name": "", "first_name": "Jane", "email": "hedgehog75@posthog.com", "redirect_url": "/", @@ -868,6 +871,7 @@ def test_api_invite_sign_up(self, mock_capture): "id": user.pk, "uuid": str(user.uuid), "distinct_id": user.distinct_id, + "last_name": "", "first_name": "Alice", "email": "test+99@posthog.com", "redirect_url": "/", @@ -1076,6 +1080,7 @@ def test_existing_user_can_sign_up_to_a_new_organization(self, mock_update_disti "id": user.pk, "uuid": str(user.uuid), "distinct_id": user.distinct_id, + "last_name": "", "first_name": "", "email": "test+159@posthog.com", "redirect_url": "/", @@ -1151,6 +1156,7 @@ def test_cannot_use_claim_invite_endpoint_to_update_user(self, mock_capture): "id": user.pk, "uuid": str(user.uuid), "distinct_id": user.distinct_id, + "last_name": "", "first_name": "", "email": "test+189@posthog.com", "redirect_url": "/", diff --git a/posthog/api/user.py b/posthog/api/user.py index c7c1813b8b38f..26e887237906b 100644 --- a/posthog/api/user.py +++ b/posthog/api/user.py @@ -89,6 +89,7 @@ class Meta: "uuid", "distinct_id", "first_name", + "last_name", "email", "pending_email", "email_opt_in", diff --git a/posthog/batch_exports/service.py b/posthog/batch_exports/service.py index d67a59becaf4a..b7e69153f0fbb 100644 --- a/posthog/batch_exports/service.py +++ b/posthog/batch_exports/service.py @@ -196,7 +196,7 @@ def pause_batch_export(temporal: Client, batch_export_id: str, note: str | None raise BatchExportServiceRPCError(f"BatchExport {batch_export_id} could not be paused") from exc batch_export.paused = True - batch_export.last_paused_at = dt.datetime.utcnow() + batch_export.last_paused_at = dt.datetime.now(dt.timezone.utc) batch_export.save() diff --git a/posthog/celery.py b/posthog/celery.py index d1804524760ac..7980df823600b 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -271,6 +271,13 @@ def setup_periodic_tasks(sender: Celery, **kwargs): name="recalculate cohorts", ) + add_periodic_task_with_expiry( + sender, + 120, + process_scheduled_changes.s(), + name="process scheduled changes", + ) + if clear_clickhouse_crontab := get_crontab(settings.CLEAR_CLICKHOUSE_REMOVED_DATA_SCHEDULE_CRON): sender.add_periodic_task( clear_clickhouse_crontab, @@ -871,6 +878,13 @@ def calculate_cohort(): calculate_cohorts() +@app.task(ignore_result=True) +def process_scheduled_changes(): + from posthog.tasks.process_scheduled_changes import process_scheduled_changes + + process_scheduled_changes() + + @app.task(ignore_result=True) def sync_insight_cache_states_task(): from posthog.caching.insight_caching_state import sync_insight_cache_states diff --git a/posthog/clickhouse/client/execute_async.py b/posthog/clickhouse/client/execute_async.py index 9be449596fdf0..065ac9640ecb4 100644 --- a/posthog/clickhouse/client/execute_async.py +++ b/posthog/clickhouse/client/execute_async.py @@ -83,7 +83,7 @@ def execute_process_query( query_status = manager.get_query_status() query_status.error = True # Assume error in case nothing below ends up working - pickup_time = datetime.datetime.utcnow() + pickup_time = datetime.datetime.now(datetime.timezone.utc) if query_status.start_time: wait_duration = (pickup_time - query_status.start_time) / datetime.timedelta(seconds=1) QUERY_WAIT_TIME.observe(wait_duration) @@ -97,7 +97,7 @@ def execute_process_query( query_status.complete = True query_status.error = False query_status.results = results - query_status.end_time = datetime.datetime.utcnow() + query_status.end_time = datetime.datetime.now(datetime.timezone.utc) query_status.expiration_time = query_status.end_time + datetime.timedelta(seconds=manager.STATUS_TTL_SECONDS) process_duration = (query_status.end_time - pickup_time) / datetime.timedelta(seconds=1) QUERY_PROCESS_TIME.observe(process_duration) @@ -131,17 +131,17 @@ def enqueue_process_query_task( return manager.get_query_status() # Immediately set status, so we don't have race with celery - query_status = QueryStatus(id=query_id, team_id=team_id, start_time=datetime.datetime.utcnow()) + query_status = QueryStatus(id=query_id, team_id=team_id, start_time=datetime.datetime.now(datetime.timezone.utc)) manager.store_query_status(query_status) if bypass_celery: # Call directly ( for testing ) process_query_task( - team_id, query_id, query_json, limit_context=LimitContext.EXPORT, refresh_requested=refresh_requested + team_id, query_id, query_json, limit_context=LimitContext.QUERY_ASYNC, refresh_requested=refresh_requested ) else: task = process_query_task.delay( - team_id, query_id, query_json, limit_context=LimitContext.EXPORT, refresh_requested=refresh_requested + team_id, query_id, query_json, limit_context=LimitContext.QUERY_ASYNC, refresh_requested=refresh_requested ) query_status.task_id = task.id manager.store_query_status(query_status) diff --git a/posthog/clickhouse/client/test/__snapshots__/test_execute_async.ambr b/posthog/clickhouse/client/test/__snapshots__/test_execute_async.ambr index 282191d2015c7..49c54a728a8bb 100644 --- a/posthog/clickhouse/client/test/__snapshots__/test_execute_async.ambr +++ b/posthog/clickhouse/client/test/__snapshots__/test_execute_async.ambr @@ -1,8 +1,8 @@ # name: ClickhouseClientTestCase.test_async_query_client ' SELECT plus(1, 1) - LIMIT 10000 SETTINGS readonly=2, - max_execution_time=600, - allow_experimental_object_type=1 + LIMIT 100 SETTINGS readonly=2, + max_execution_time=600, + allow_experimental_object_type=1 ' --- diff --git a/posthog/clickhouse/migrations/0051_session_replay_source_mv_fix.py b/posthog/clickhouse/migrations/0051_session_replay_source_mv_fix.py new file mode 100644 index 0000000000000..4ac35e2535f92 --- /dev/null +++ b/posthog/clickhouse/migrations/0051_session_replay_source_mv_fix.py @@ -0,0 +1,15 @@ +from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions +from posthog.session_recordings.sql.session_replay_event_migrations_sql import ( + DROP_SESSION_REPLAY_EVENTS_TABLE_MV_SQL, +) +from posthog.session_recordings.sql.session_replay_event_sql import ( + SESSION_REPLAY_EVENTS_TABLE_MV_SQL, +) + +operations = [ + # we have to drop materialized view because 0050 created it incorrectly + run_sql_with_exceptions(DROP_SESSION_REPLAY_EVENTS_TABLE_MV_SQL()), + # now we can recreate it with explicit column definitions + # that correctly identifies snapshot source as LowCardinality(Nullable(String)) + run_sql_with_exceptions(SESSION_REPLAY_EVENTS_TABLE_MV_SQL()), +] diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr index 6e4344727633b..d337f3d55201e 100644 --- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr +++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr @@ -1495,7 +1495,19 @@ ' CREATE MATERIALIZED VIEW IF NOT EXISTS session_replay_events_mv ON CLUSTER 'posthog' - TO posthog_test.writable_session_replay_events + TO posthog_test.writable_session_replay_events ( + `session_id` String, `team_id` Int64, `distinct_id` String, + `min_first_timestamp` DateTime64(6, 'UTC'), + `max_last_timestamp` DateTime64(6, 'UTC'), + `first_url` AggregateFunction(argMin, Nullable(String), DateTime64(6, 'UTC')), + `click_count` Int64, `keypress_count` Int64, + `mouse_activity_count` Int64, `active_milliseconds` Int64, + `console_log_count` Int64, `console_warn_count` Int64, + `console_error_count` Int64, `size` Int64, `message_count` Int64, + `event_count` Int64, + `snapshot_source` AggregateFunction(argMin, LowCardinality(Nullable(String)), DateTime64(6, 'UTC')), + `_timestamp` Nullable(DateTime) + ) AS SELECT session_id, team_id, diff --git a/posthog/clickhouse/test/test_person_overrides.py b/posthog/clickhouse/test/test_person_overrides.py index dc8bc2b17c503..ec632eebe7774 100644 --- a/posthog/clickhouse/test/test_person_overrides.py +++ b/posthog/clickhouse/test/test_person_overrides.py @@ -1,5 +1,5 @@ import json -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from time import sleep from typing import TypedDict from uuid import UUID, uuid4 @@ -124,7 +124,7 @@ def test_person_overrides_dict(): "override_person_id": uuid4(), "merged_at": datetime.fromisoformat("2020-01-02T00:00:00+00:00"), "oldest_event": datetime.fromisoformat("2020-01-01T00:00:00+00:00"), - "created_at": datetime.utcnow(), + "created_at": datetime.now(timezone.utc), "version": 1, } diff --git a/posthog/constants.py b/posthog/constants.py index d0b1d4a6e0c52..3110b3ff2da9c 100644 --- a/posthog/constants.py +++ b/posthog/constants.py @@ -204,6 +204,7 @@ class AvailableFeature(str, Enum): PATH_MAX_EDGE_WEIGHT = "max_edge_weight" AGGREGATION_GROUP_TYPE_INDEX = "aggregation_group_type_index" BREAKDOWN_HISTOGRAM_BIN_COUNT = "breakdown_histogram_bin_count" +BREAKDOWN_HIDE_OTHER_AGGREGATION = "breakdown_hide_other_aggregation" BREAKDOWN_NORMALIZE_URL = "breakdown_normalize_url" SAMPLING_FACTOR = "sampling_factor" diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index 75b6c0fc63342..ef5a0e587380c 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -39,6 +39,7 @@ class LimitContext(str, Enum): QUERY = "query" + QUERY_ASYNC = "query_async" EXPORT = "export" COHORT_CALCULATION = "cohort_calculation" @@ -46,7 +47,7 @@ class LimitContext(str, Enum): def get_max_limit_for_context(limit_context: LimitContext) -> int: if limit_context == LimitContext.EXPORT: return MAX_SELECT_RETURNED_ROWS # 10k - elif limit_context == LimitContext.QUERY: + elif limit_context in (LimitContext.QUERY, LimitContext.QUERY_ASYNC): return MAX_SELECT_RETURNED_ROWS # 10k elif limit_context == LimitContext.COHORT_CALCULATION: return MAX_SELECT_COHORT_CALCULATION_LIMIT # 1b @@ -58,7 +59,7 @@ def get_default_limit_for_context(limit_context: LimitContext) -> int: """Limit used if no limit is provided""" if limit_context == LimitContext.EXPORT: return MAX_SELECT_RETURNED_ROWS # 10k - elif limit_context == LimitContext.QUERY: + elif limit_context in (LimitContext.QUERY, LimitContext.QUERY_ASYNC): return DEFAULT_RETURNED_ROWS # 100 elif limit_context == LimitContext.COHORT_CALCULATION: return MAX_SELECT_COHORT_CALCULATION_LIMIT # 1b diff --git a/posthog/hogql/database/schema/session_replay_events.py b/posthog/hogql/database/schema/session_replay_events.py index f163e8052e8bf..7a2097c2b8b5f 100644 --- a/posthog/hogql/database/schema/session_replay_events.py +++ b/posthog/hogql/database/schema/session_replay_events.py @@ -17,6 +17,8 @@ ) from posthog.schema import HogQLQueryModifiers +RAW_ONLY_FIELDS = ["min_first_timestamp", "max_last_timestamp"] + SESSION_REPLAY_EVENTS_COMMON_FIELDS: Dict[str, FieldOrTable] = { "session_id": StringDatabaseField(name="session_id"), "team_id": IntegerDatabaseField(name="team_id"), @@ -88,6 +90,10 @@ def select_from_session_replay_events_table(requested_fields: Dict[str, List[str group_by_fields: List[ast.Expr] = [] for name, chain in requested_fields.items(): + if name in RAW_ONLY_FIELDS: + # these fields are accounted for by start_time and end_time, so can be skipped in the "not raw" table + continue + if name in aggregate_fields: select_fields.append(ast.Alias(alias=name, expr=aggregate_fields[name])) else: @@ -103,7 +109,7 @@ def select_from_session_replay_events_table(requested_fields: Dict[str, List[str class SessionReplayEventsTable(LazyTable): fields: Dict[str, FieldOrTable] = { - **SESSION_REPLAY_EVENTS_COMMON_FIELDS, + **{k: v for k, v in SESSION_REPLAY_EVENTS_COMMON_FIELDS.items() if k not in RAW_ONLY_FIELDS}, "start_time": DateTimeDatabaseField(name="start_time"), "end_time": DateTimeDatabaseField(name="end_time"), "first_url": StringDatabaseField(name="first_url"), diff --git a/posthog/hogql/database/test/__snapshots__/test_database.ambr b/posthog/hogql/database/test/__snapshots__/test_database.ambr index 2584418634a73..c03b07daa0df3 100644 --- a/posthog/hogql/database/test/__snapshots__/test_database.ambr +++ b/posthog/hogql/database/test/__snapshots__/test_database.ambr @@ -320,14 +320,6 @@ "key": "distinct_id", "type": "string" }, - { - "key": "min_first_timestamp", - "type": "datetime" - }, - { - "key": "max_last_timestamp", - "type": "datetime" - }, { "key": "first_url", "type": "string" @@ -1089,14 +1081,6 @@ "key": "distinct_id", "type": "string" }, - { - "key": "min_first_timestamp", - "type": "datetime" - }, - { - "key": "max_last_timestamp", - "type": "datetime" - }, { "key": "first_url", "type": "string" diff --git a/posthog/hogql/database/test/test_database.py b/posthog/hogql/database/test/test_database.py index 0657779b15b87..c0817ee2c9861 100644 --- a/posthog/hogql/database/test/test_database.py +++ b/posthog/hogql/database/test/test_database.py @@ -70,7 +70,7 @@ def test_database_with_warehouse_tables(self, patch_execute): self.assertEqual( response.clickhouse, - f"SELECT whatever.id FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_3_sensitive)s, %(hogql_val_4_sensitive)s, %(hogql_val_1)s, %(hogql_val_2)s) AS whatever LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", + f"SELECT whatever.id AS id FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_3_sensitive)s, %(hogql_val_4_sensitive)s, %(hogql_val_1)s, %(hogql_val_2)s) AS whatever LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", ) def test_database_group_type_mappings(self): @@ -100,17 +100,20 @@ def test_database_expression_fields(self): query = print_ast(parse_select(sql), context, dialect="clickhouse") assert ( query - == "SELECT numbers.number, multiply(numbers.number, 2) AS double, plus(plus(1, 1), numbers.number) FROM numbers(2) AS numbers LIMIT 10000" + == "SELECT numbers.number AS number, multiply(numbers.number, 2) AS double, plus(plus(1, 1), numbers.number) FROM numbers(2) AS numbers LIMIT 10000" ), query sql = "select double from (select double from numbers(2))" query = print_ast(parse_select(sql), context, dialect="clickhouse") assert ( query - == "SELECT double FROM (SELECT multiply(numbers.number, 2) AS double FROM numbers(2) AS numbers) LIMIT 10000" + == "SELECT double AS double FROM (SELECT multiply(numbers.number, 2) AS double FROM numbers(2) AS numbers) LIMIT 10000" ), query # expression fields are not included in select * sql = "select * from (select * from numbers(2))" query = print_ast(parse_select(sql), context, dialect="clickhouse") - assert query == "SELECT number FROM (SELECT numbers.number FROM numbers(2) AS numbers) LIMIT 10000", query + assert ( + query + == "SELECT number AS number FROM (SELECT numbers.number AS number FROM numbers(2) AS numbers) LIMIT 10000" + ), query diff --git a/posthog/hogql/database/test/test_s3_table.py b/posthog/hogql/database/test/test_s3_table.py index 72b5dfa6cf3c0..dae4b7aec90be 100644 --- a/posthog/hogql/database/test/test_s3_table.py +++ b/posthog/hogql/database/test/test_s3_table.py @@ -36,7 +36,7 @@ def test_s3_table_select(self): self.assertEqual( clickhouse, - "SELECT aapl_stock.Date, aapl_stock.Open, aapl_stock.High, aapl_stock.Low, aapl_stock.Close, aapl_stock.Volume, aapl_stock.OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock LIMIT 10", + "SELECT aapl_stock.Date AS Date, aapl_stock.Open AS Open, aapl_stock.High AS High, aapl_stock.Low AS Low, aapl_stock.Close AS Close, aapl_stock.Volume AS Volume, aapl_stock.OpenInt AS OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock LIMIT 10", ) def test_s3_table_select_with_alias(self): @@ -50,7 +50,7 @@ def test_s3_table_select_with_alias(self): # Alias will completely override table name to prevent ambiguous table names that can be shared if the same table is joinedfrom multiple times self.assertEqual( clickhouse, - "SELECT a.High, a.Low FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS a LIMIT 10", + "SELECT a.High AS High, a.Low AS Low FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS a LIMIT 10", ) def test_s3_table_select_join(self): @@ -72,7 +72,7 @@ def test_s3_table_select_join(self): self.assertEqual( clickhouse, - "SELECT aapl_stock.High, aapl_stock.Low FROM (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS aapl_stock JOIN (SELECT * FROM s3Cluster('posthog', %(hogql_val_3_sensitive)s, %(hogql_val_4)s)) AS aapl_stock_2 ON equals(aapl_stock.High, aapl_stock_2.High) LIMIT 10", + "SELECT aapl_stock.High AS High, aapl_stock.Low AS Low FROM (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS aapl_stock JOIN (SELECT * FROM s3Cluster('posthog', %(hogql_val_3_sensitive)s, %(hogql_val_4)s)) AS aapl_stock_2 ON equals(aapl_stock.High, aapl_stock_2.High) LIMIT 10", ) def test_s3_table_select_join_with_alias(self): @@ -95,7 +95,7 @@ def test_s3_table_select_join_with_alias(self): # Alias will completely override table name to prevent ambiguous table names that can be shared if the same table is joinedfrom multiple times self.assertEqual( clickhouse, - "SELECT a.High, a.Low FROM (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS a JOIN (SELECT * FROM s3Cluster('posthog', %(hogql_val_3_sensitive)s, %(hogql_val_4)s)) AS b ON equals(a.High, b.High) LIMIT 10", + "SELECT a.High AS High, a.Low AS Low FROM (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS a JOIN (SELECT * FROM s3Cluster('posthog', %(hogql_val_3_sensitive)s, %(hogql_val_4)s)) AS b ON equals(a.High, b.High) LIMIT 10", ) def test_s3_table_select_and_non_s3_join(self): @@ -117,7 +117,7 @@ def test_s3_table_select_and_non_s3_join(self): self.assertEqual( clickhouse, - f"SELECT aapl_stock.High, aapl_stock.Low FROM (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS aapl_stock JOIN events ON equals(aapl_stock.High, events.event) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"SELECT aapl_stock.High AS High, aapl_stock.Low AS Low FROM (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS aapl_stock JOIN events ON equals(aapl_stock.High, events.event) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", ) def test_s3_table_select_and_non_s3_join_first(self): @@ -139,7 +139,7 @@ def test_s3_table_select_and_non_s3_join_first(self): self.assertEqual( clickhouse, - f"SELECT aapl_stock.High, aapl_stock.Low FROM events GLOBAL JOIN (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS aapl_stock ON equals(aapl_stock.High, events.event) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"SELECT aapl_stock.High AS High, aapl_stock.Low AS Low FROM events GLOBAL JOIN (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS aapl_stock ON equals(aapl_stock.High, events.event) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", ) def test_s3_table_select_alias_escaped(self): @@ -167,7 +167,7 @@ def test_s3_table_select_alias_escaped(self): # table name is escaped self.assertEqual( clickhouse, - f"SELECT `random as (SELECT * FROM events), SELECT * FROM events --`.High, `random as (SELECT * FROM events), SELECT * FROM events --`.Low FROM (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS `random as (SELECT * FROM events), SELECT * FROM events --` JOIN events ON equals(`random as (SELECT * FROM events), SELECT * FROM events --`.High, events.event) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"SELECT `random as (SELECT * FROM events), SELECT * FROM events --`.High AS High, `random as (SELECT * FROM events), SELECT * FROM events --`.Low AS Low FROM (SELECT * FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s)) AS `random as (SELECT * FROM events), SELECT * FROM events --` JOIN events ON equals(`random as (SELECT * FROM events), SELECT * FROM events --`.High, events.event) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", ) def test_s3_table_select_table_name_bad_character(self): @@ -199,5 +199,5 @@ def test_s3_table_select_in(self): self.assertEqual( clickhouse, - f"SELECT events.uuid, events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), ifNull(globalIn(events.event, (SELECT aapl_stock.Date FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock)), 0)) LIMIT 10000", + f"SELECT events.uuid AS uuid, events.event AS event FROM events WHERE and(equals(events.team_id, {self.team.pk}), ifNull(globalIn(events.event, (SELECT aapl_stock.Date AS Date FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock)), 0)) LIMIT 10000", ) diff --git a/posthog/hogql/database/test/test_saved_query.py b/posthog/hogql/database/test/test_saved_query.py index 7c7f534c66f21..7369c50a29181 100644 --- a/posthog/hogql/database/test/test_saved_query.py +++ b/posthog/hogql/database/test/test_saved_query.py @@ -44,7 +44,7 @@ def test_saved_query_table_select(self): self.assertEqual( clickhouse, - "SELECT aapl_stock_view.Date, aapl_stock_view.Open, aapl_stock_view.High, aapl_stock_view.Low, aapl_stock_view.Close, aapl_stock_view.Volume, aapl_stock_view.OpenInt FROM (SELECT aapl_stock.Date, aapl_stock.Open, aapl_stock.High, aapl_stock.Low, aapl_stock.Close, aapl_stock.Volume, aapl_stock.OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock) AS aapl_stock_view LIMIT 10", + "SELECT aapl_stock_view.Date AS Date, aapl_stock_view.Open AS Open, aapl_stock_view.High AS High, aapl_stock_view.Low AS Low, aapl_stock_view.Close AS Close, aapl_stock_view.Volume AS Volume, aapl_stock_view.OpenInt AS OpenInt FROM (SELECT aapl_stock.Date AS Date, aapl_stock.Open AS Open, aapl_stock.High AS High, aapl_stock.Low AS Low, aapl_stock.Close AS Close, aapl_stock.Volume AS Volume, aapl_stock.OpenInt AS OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock) AS aapl_stock_view LIMIT 10", ) def test_saved_query_with_alias(self): @@ -63,5 +63,5 @@ def test_saved_query_with_alias(self): self.assertEqual( clickhouse, - "SELECT some_alias.Date, some_alias.Open, some_alias.High, some_alias.Low, some_alias.Close, some_alias.Volume, some_alias.OpenInt FROM (SELECT aapl_stock.Date, aapl_stock.Open, aapl_stock.High, aapl_stock.Low, aapl_stock.Close, aapl_stock.Volume, aapl_stock.OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock) AS some_alias LIMIT 10", + "SELECT some_alias.Date AS Date, some_alias.Open AS Open, some_alias.High AS High, some_alias.Low AS Low, some_alias.Close AS Close, some_alias.Volume AS Volume, some_alias.OpenInt AS OpenInt FROM (SELECT aapl_stock.Date AS Date, aapl_stock.Open AS Open, aapl_stock.High AS High, aapl_stock.Low AS Low, aapl_stock.Close AS Close, aapl_stock.Volume AS Volume, aapl_stock.OpenInt AS OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock) AS some_alias LIMIT 10", ) diff --git a/posthog/hogql/database/test/test_view.py b/posthog/hogql/database/test/test_view.py index 26ce89e10653c..e6700d4bbc0f0 100644 --- a/posthog/hogql/database/test/test_view.py +++ b/posthog/hogql/database/test/test_view.py @@ -44,7 +44,12 @@ def test_view_table_select(self): self.assertEqual( clickhouse, - "SELECT aapl_stock_view.Date, aapl_stock_view.Open, aapl_stock_view.High, aapl_stock_view.Low, aapl_stock_view.Close, aapl_stock_view.Volume, aapl_stock_view.OpenInt FROM (SELECT aapl_stock.Date, aapl_stock.Open, aapl_stock.High, aapl_stock.Low, aapl_stock.Close, aapl_stock.Volume, aapl_stock.OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock) AS aapl_stock_view LIMIT 10", + "SELECT aapl_stock_view.Date AS Date, aapl_stock_view.Open AS Open, aapl_stock_view.High AS High, " + "aapl_stock_view.Low AS Low, aapl_stock_view.Close AS Close, aapl_stock_view.Volume AS Volume, " + "aapl_stock_view.OpenInt AS OpenInt FROM (SELECT aapl_stock.Date AS Date, aapl_stock.Open AS Open, " + "aapl_stock.High AS High, aapl_stock.Low AS Low, aapl_stock.Close AS Close, aapl_stock.Volume AS Volume, " + "aapl_stock.OpenInt AS OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock) " + "AS aapl_stock_view LIMIT 10", ) def test_view_with_alias(self): @@ -63,5 +68,5 @@ def test_view_with_alias(self): self.assertEqual( clickhouse, - "SELECT some_alias.Date, some_alias.Open, some_alias.High, some_alias.Low, some_alias.Close, some_alias.Volume, some_alias.OpenInt FROM (SELECT aapl_stock.Date, aapl_stock.Open, aapl_stock.High, aapl_stock.Low, aapl_stock.Close, aapl_stock.Volume, aapl_stock.OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock) AS some_alias LIMIT 10", + "SELECT some_alias.Date AS Date, some_alias.Open AS Open, some_alias.High AS High, some_alias.Low AS Low, some_alias.Close AS Close, some_alias.Volume AS Volume, some_alias.OpenInt AS OpenInt FROM (SELECT aapl_stock.Date AS Date, aapl_stock.Open AS Open, aapl_stock.High AS High, aapl_stock.Low AS Low, aapl_stock.Close AS Close, aapl_stock.Volume AS Volume, aapl_stock.OpenInt AS OpenInt FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_1)s) AS aapl_stock) AS some_alias LIMIT 10", ) diff --git a/posthog/hogql/functions/test/__snapshots__/test_cohort.ambr b/posthog/hogql/functions/test/__snapshots__/test_cohort.ambr new file mode 100644 index 0000000000000..b44ac2b6f0f09 --- /dev/null +++ b/posthog/hogql/functions/test/__snapshots__/test_cohort.ambr @@ -0,0 +1,76 @@ +# name: TestCohort.test_in_cohort_dynamic + ' + -- ClickHouse + + SELECT events.event AS event + FROM events + WHERE and(equals(events.team_id, 420), in(events.person_id, ( + SELECT cohortpeople.person_id AS person_id + FROM cohortpeople + WHERE and(equals(cohortpeople.team_id, 420), equals(cohortpeople.cohort_id, 6)) + GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version + HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), equals(events.event, %(hogql_val_0)s)) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events + WHERE and(in(person_id, ( + SELECT person_id + FROM raw_cohort_people + WHERE equals(cohort_id, 6) + GROUP BY person_id, cohort_id, version + HAVING greater(sum(sign), 0))), equals(event, 'RANDOM_TEST_ID::UUID')) + LIMIT 100 + ' +--- +# name: TestCohort.test_in_cohort_static + ' + -- ClickHouse + + SELECT events.event AS event + FROM events + WHERE and(equals(events.team_id, 420), in(events.person_id, ( + SELECT person_static_cohort.person_id AS person_id + FROM person_static_cohort + WHERE and(equals(person_static_cohort.team_id, 420), equals(person_static_cohort.cohort_id, 7))))) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events + WHERE in(person_id, ( + SELECT person_id + FROM static_cohort_people + WHERE equals(cohort_id, 7))) + LIMIT 100 + ' +--- +# name: TestCohort.test_in_cohort_strings + ' + -- ClickHouse + + SELECT events.event AS event + FROM events + WHERE and(equals(events.team_id, 420), in(events.person_id, ( + SELECT person_static_cohort.person_id AS person_id + FROM person_static_cohort + WHERE and(equals(person_static_cohort.team_id, 420), equals(person_static_cohort.cohort_id, 8))))) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events + WHERE in(person_id, ( + SELECT person_id + FROM static_cohort_people + WHERE equals(cohort_id, 8))) + LIMIT 100 + ' +--- diff --git a/posthog/hogql/functions/test/test_cohort.py b/posthog/hogql/functions/test/test_cohort.py index 0f03d28fcd6e5..1c5460bc9176d 100644 --- a/posthog/hogql/functions/test/test_cohort.py +++ b/posthog/hogql/functions/test/test_cohort.py @@ -1,9 +1,11 @@ +import pytest from django.test import override_settings from posthog.hogql import ast from posthog.hogql.errors import HogQLException from posthog.hogql.parser import parse_expr from posthog.hogql.query import execute_hogql_query +from posthog.hogql.test.utils import pretty_print_response_in_tests from posthog.models import Cohort from posthog.models.cohort.util import recalculate_cohortpeople from posthog.models.utils import UUIDT @@ -23,7 +25,7 @@ class TestCohort(BaseTest): maxDiff = None def _create_random_events(self) -> str: - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" _create_person( properties={"$os": "Chrome", "random_uuid": random_uuid}, team=self.team, @@ -34,6 +36,7 @@ def _create_random_events(self) -> str: flush_persons_and_events() return random_uuid + @pytest.mark.usefixtures("unittest_snapshot") @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=False) def test_in_cohort_dynamic(self): random_uuid = self._create_random_events() @@ -47,17 +50,11 @@ def test_in_cohort_dynamic(self): self.team, modifiers=HogQLQueryModifiers(inCohortVia="subquery"), ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT cohortpeople.person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), equals(events.event, %(hogql_val_0)s)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) - self.assertEqual( - response.hogql, - f"SELECT event FROM events WHERE and(in(person_id, (SELECT person_id FROM raw_cohort_people WHERE equals(cohort_id, {cohort.pk}) GROUP BY person_id, cohort_id, version HAVING greater(sum(sign), 0))), equals(event, '{random_uuid}')) LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(len(response.results), 1) self.assertEqual(response.results[0][0], random_uuid) + @pytest.mark.usefixtures("unittest_snapshot") @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=False) def test_in_cohort_static(self): cohort = Cohort.objects.create( @@ -69,18 +66,12 @@ def test_in_cohort_static(self): self.team, modifiers=HogQLQueryModifiers(inCohortVia="subquery"), ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) - self.assertEqual( - response.hogql, - f"SELECT event FROM events WHERE in(person_id, (SELECT person_id FROM static_cohort_people WHERE equals(cohort_id, {cohort.pk}))) LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + @pytest.mark.usefixtures("unittest_snapshot") @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=False) def test_in_cohort_strings(self): - cohort = Cohort.objects.create( + Cohort.objects.create( team=self.team, name="my cohort", is_static=True, @@ -90,14 +81,7 @@ def test_in_cohort_strings(self): self.team, modifiers=HogQLQueryModifiers(inCohortVia="subquery"), ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) - self.assertEqual( - response.hogql, - f"SELECT event FROM events WHERE in(person_id, (SELECT person_id FROM static_cohort_people WHERE equals(cohort_id, {cohort.pk}))) LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=True) def test_in_cohort_error(self): diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 21f9a68bab22e..acade9b195878 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -236,39 +236,28 @@ def visit_select_query(self, node: ast.SelectQuery): next_join = next_join.next_join if node.select: - # Only for ClickHouse: Gather all visible aliases, and/or the last hidden alias for - # each unique alias name. Then make the last hidden aliases visible. if self.dialect == "clickhouse": - visible_aliases = {} + # Gather all visible aliases, and/or the last hidden alias for each unique alias name. + found_aliases = {} for alias in reversed(node.select): if isinstance(alias, ast.Alias): - if not visible_aliases.get(alias.alias, None) or not alias.hidden: - visible_aliases[alias.alias] = alias + if not found_aliases.get(alias.alias, None) or not alias.hidden: + found_aliases[alias.alias] = alias columns = [] for column in node.select: if isinstance(column, ast.Alias): - # It's either a visible alias, or the last hidden alias for this name. - if visible_aliases.get(column.alias) == column: + # It's either a visible alias, or the last hidden alias with this name. + if found_aliases.get(column.alias) == column: if column.hidden: - if ( - isinstance(column.expr, ast.Field) - and isinstance(column.expr.type, ast.FieldType) - and column.expr.type.name == column.alias - ): - # Hide the hidden alias only if it's a simple field, - # and we're using the same name for the field and the alias - # E.g. events.event AS event --> events.evnet. - column = column.expr - else: - # Make the hidden alias visible - column = cast(ast.Alias, clone_expr(column)) - column.hidden = False + # Make the hidden alias visible + column = cast(ast.Alias, clone_expr(column)) + column.hidden = False else: # Always print visible aliases. pass else: - # This is not the alias for this unique alias name. Skip it. + # Non-unique hidden alias. Skip. column = column.expr columns.append(self.visit(column)) else: diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py index 767e9f99d7b54..63b99ea516f5e 100644 --- a/posthog/hogql/query.py +++ b/posthog/hogql/query.py @@ -22,7 +22,7 @@ from posthog.client import sync_execute from posthog.schema import HogQLQueryResponse, HogQLFilters, HogQLQueryModifiers -EXPORT_CONTEXT_MAX_EXECUTION_TIME = 600 +INCREASED_MAX_EXECUTION_TIME = 600 def execute_hogql_query( @@ -114,8 +114,8 @@ def execute_hogql_query( ) settings = settings or HogQLGlobalSettings() - if limit_context == LimitContext.EXPORT or limit_context == LimitContext.COHORT_CALCULATION: - settings.max_execution_time = EXPORT_CONTEXT_MAX_EXECUTION_TIME + if limit_context in (LimitContext.EXPORT, LimitContext.COHORT_CALCULATION, LimitContext.QUERY_ASYNC): + settings.max_execution_time = INCREASED_MAX_EXECUTION_TIME # Print the ClickHouse SQL query with timings.measure("print_ast"): diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 9d69cd9c2954d..6a2292be47e50 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -141,6 +141,8 @@ def visit_select_query(self, node: ast.SelectQuery): alias = new_expr.type.alias elif isinstance(new_expr.type, ast.FieldType): alias = new_expr.type.name + elif isinstance(new_expr.type, ast.ExpressionFieldType): + alias = new_expr.type.name elif isinstance(new_expr, ast.Alias): alias = new_expr.alias else: @@ -444,6 +446,7 @@ def visit_field(self, node: ast.Field): raise ResolverException(f"Unable to resolve field: {name}") # Recursively resolve the rest of the chain until we can point to the deepest node. + field_name = node.chain[-1] loop_type = type chain_to_parse = node.chain[1:] previous_types = [] @@ -484,7 +487,7 @@ def visit_field(self, node: ast.Field): if isinstance(node.type, ast.FieldType): return ast.Alias( - alias=node.type.name, + alias=field_name or node.type.name, expr=node, hidden=True, type=ast.FieldAliasType(alias=node.type.name, type=node.type), diff --git a/posthog/hogql/test/__snapshots__/test_query.ambr b/posthog/hogql/test/__snapshots__/test_query.ambr index d656bc1eec6b2..44499a927d566 100644 --- a/posthog/hogql/test/__snapshots__/test_query.ambr +++ b/posthog/hogql/test/__snapshots__/test_query.ambr @@ -1,63 +1,387 @@ # name: TestQuery.test_hogql_arrays ' + -- ClickHouse SELECT [1, 2, 3], [10, 11, 12][1] LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT [1, 2, 3], [10, 11, 12][1] + LIMIT 100 ' --- # name: TestQuery.test_hogql_lambdas ' + -- ClickHouse SELECT arrayMap(x -> multiply(x, 2), [1, 2, 3]), 1 LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT arrayMap(x -> multiply(x, 2), [1, 2, 3]), 1 + LIMIT 100 + ' +--- +# name: TestQuery.test_hogql_query_filters + ' + -- ClickHouse + + SELECT events.event AS event, events.distinct_id AS distinct_id + FROM events + WHERE and(equals(events.team_id, 420), equals(events.distinct_id, %(hogql_val_0)s), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0)) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, distinct_id + FROM events + WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), equals(properties.index, '4')) + LIMIT 100 + ' +--- +# name: TestQuery.test_hogql_query_filters.1 + ' + + SELECT event, distinct_id + FROM events + WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), equals(properties.index, '4')) + LIMIT 100 + ' +--- +# name: TestQuery.test_hogql_query_filters.2 + ' + -- ClickHouse + + SELECT events.event AS event, events.distinct_id AS distinct_id + FROM events + WHERE and(equals(events.team_id, 420), equals(events.distinct_id, %(hogql_val_0)s), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0), less(toTimeZone(events.timestamp, %(hogql_val_3)s), toDateTime64('2020-01-02 00:00:00.000000', 6, 'UTC')), greaterOrEquals(toTimeZone(events.timestamp, %(hogql_val_4)s), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')))) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, distinct_id + FROM events + WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), and(equals(properties.index, '4'), less(timestamp, toDateTime('2020-01-02 00:00:00.000000')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) + LIMIT 100 + ' +--- +# name: TestQuery.test_hogql_query_filters.3 + ' + + SELECT event, distinct_id + FROM events + WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), and(equals(properties.index, '4'), less(timestamp, toDateTime('2020-01-02 00:00:00.000000')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) + LIMIT 100 ' --- # name: TestQuery.test_hogql_query_filters_alias ' + -- ClickHouse - SELECT e.event, e.distinct_id + SELECT e.event AS event, e.distinct_id AS distinct_id FROM events AS e WHERE and(equals(e.team_id, 420), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), 0)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, distinct_id + FROM events AS e + WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') + LIMIT 100 ' --- # name: TestQuery.test_hogql_union_all_limits ' + -- ClickHouse - SELECT events.event + SELECT events.event AS event FROM events WHERE equals(events.team_id, 420) LIMIT 100 UNION ALL - SELECT events.event + SELECT events.event AS event FROM events WHERE equals(events.team_id, 420) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events + LIMIT 100 UNION ALL + SELECT event + FROM events + LIMIT 100 + ' +--- +# name: TestQuery.test_join_with_property_materialized_session_id + ' + -- ClickHouse + + SELECT e.event AS event, s.session_id AS session_id + FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, nullIf(nullIf(e.`$session_id`, ''), 'null')) + WHERE and(equals(s.team_id, 420), equals(e.team_id, 420), isNotNull(nullIf(nullIf(e.`$session_id`, ''), 'null'))) + LIMIT 10 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT e.event, s.session_id + FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, e.properties.$session_id) + WHERE notEquals(e.properties.$session_id, NULL) + LIMIT 10 + ' +--- +# name: TestQuery.test_join_with_property_materialized_session_id.1 + ' + + SELECT e.event, s.session_id + FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, e.properties.$session_id) + WHERE notEquals(e.properties.$session_id, NULL) + LIMIT 10 + ' +--- +# name: TestQuery.test_join_with_property_materialized_session_id.2 + ' + -- ClickHouse + + SELECT e.event AS event, s.session_id AS session_id + FROM session_replay_events AS s LEFT JOIN events AS e ON equals(nullIf(nullIf(e.`$session_id`, ''), 'null'), s.session_id) + WHERE and(equals(e.team_id, 420), equals(s.team_id, 420), isNotNull(nullIf(nullIf(e.`$session_id`, ''), 'null'))) + LIMIT 10 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT e.event, s.session_id + FROM session_replay_events AS s LEFT JOIN events AS e ON equals(e.properties.$session_id, s.session_id) + WHERE notEquals(e.properties.$session_id, NULL) + LIMIT 10 + ' +--- +# name: TestQuery.test_join_with_property_materialized_session_id.3 + ' + + SELECT e.event, s.session_id + FROM session_replay_events AS s LEFT JOIN events AS e ON equals(e.properties.$session_id, s.session_id) + WHERE notEquals(e.properties.$session_id, NULL) + LIMIT 10 + ' +--- +# name: TestQuery.test_join_with_property_not_materialized + ' + -- ClickHouse + + SELECT e.event AS event, s.session_id AS session_id + FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')) + WHERE and(equals(s.team_id, 420), equals(e.team_id, 420), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''))) + LIMIT 10 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT e.event, s.session_id + FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, e.properties.$$$session_id) + WHERE notEquals(e.properties.$$$session_id, NULL) + LIMIT 10 + ' +--- +# name: TestQuery.test_join_with_property_not_materialized.1 + ' + + SELECT e.event, s.session_id + FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, e.properties.$$$session_id) + WHERE notEquals(e.properties.$$$session_id, NULL) + LIMIT 10 + ' +--- +# name: TestQuery.test_join_with_property_not_materialized.2 + ' + -- ClickHouse + + SELECT e.event AS event, s.session_id AS session_id + FROM session_replay_events AS s LEFT JOIN events AS e ON equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), s.session_id) + WHERE and(equals(e.team_id, 420), equals(s.team_id, 420), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''))) + LIMIT 10 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT e.event, s.session_id + FROM session_replay_events AS s LEFT JOIN events AS e ON equals(e.properties.$$$session_id, s.session_id) + WHERE notEquals(e.properties.$$$session_id, NULL) + LIMIT 10 + ' +--- +# name: TestQuery.test_join_with_property_not_materialized.3 + ' + + SELECT e.event, s.session_id + FROM session_replay_events AS s LEFT JOIN events AS e ON equals(e.properties.$$$session_id, s.session_id) + WHERE notEquals(e.properties.$$$session_id, NULL) + LIMIT 10 + ' +--- +# name: TestQuery.test_prop_cohort_basic + ' + -- ClickHouse + + SELECT events.event AS event, count() + FROM events INNER JOIN ( + SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 420) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) + WHERE and(equals(events.team_id, 420), ifNull(in(events__pdi.person_id, ( + SELECT cohortpeople.person_id AS person_id + FROM cohortpeople + WHERE and(equals(cohortpeople.team_id, 420), equals(cohortpeople.cohort_id, 13)) + GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version + HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)) + GROUP BY events.event + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, count() + FROM events + WHERE in(person_id, ( + SELECT person_id + FROM raw_cohort_people + WHERE equals(cohort_id, 13) + GROUP BY person_id, cohort_id, version + HAVING greater(sum(sign), 0))) + GROUP BY event + LIMIT 100 + ' +--- +# name: TestQuery.test_prop_cohort_basic.1 + ' + -- ClickHouse + + SELECT events.event AS event, count(*) + FROM events + WHERE and(equals(events.team_id, 420), in(events.person_id, ( + SELECT cohortpeople.person_id AS person_id + FROM cohortpeople + WHERE and(equals(cohortpeople.team_id, 420), equals(cohortpeople.cohort_id, 13)) + GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version + HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0)))) + GROUP BY events.event + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, count(*) + FROM events + WHERE in(person_id, ( + SELECT person_id + FROM raw_cohort_people + WHERE equals(cohort_id, 13) + GROUP BY person_id, cohort_id, version + HAVING greater(sum(sign), 0))) + GROUP BY event + LIMIT 100 + ' +--- +# name: TestQuery.test_prop_cohort_static + ' + -- ClickHouse + + SELECT events.event AS event, count() + FROM events INNER JOIN ( + SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 420) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) + WHERE and(equals(events.team_id, 420), ifNull(in(events__pdi.person_id, ( + SELECT person_static_cohort.person_id AS person_id + FROM person_static_cohort + WHERE and(equals(person_static_cohort.team_id, 420), equals(person_static_cohort.cohort_id, 14)))), 0)) + GROUP BY events.event + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, count() + FROM events + WHERE in(person_id, ( + SELECT person_id + FROM static_cohort_people + WHERE equals(cohort_id, 14))) + GROUP BY event + LIMIT 100 + ' +--- +# name: TestQuery.test_prop_cohort_static.1 + ' + -- ClickHouse + + SELECT events.event AS event, count(*) + FROM events + WHERE and(equals(events.team_id, 420), in(events.person_id, ( + SELECT person_static_cohort.person_id AS person_id + FROM person_static_cohort + WHERE and(equals(person_static_cohort.team_id, 420), equals(person_static_cohort.cohort_id, 14))))) + GROUP BY events.event + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, count(*) + FROM events + WHERE in(person_id, ( + SELECT person_id + FROM static_cohort_people + WHERE equals(cohort_id, 14))) + GROUP BY event + LIMIT 100 ' --- # name: TestQuery.test_query ' + -- ClickHouse - SELECT count(), events.event + SELECT count(), events.event AS event FROM events WHERE and(equals(events.team_id, 420), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), 0)) GROUP BY events.event LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT count(), event + FROM events + WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') + GROUP BY event + LIMIT 100 ' --- # name: TestQuery.test_query_distinct ' + -- ClickHouse SELECT DISTINCT persons.properties___sneaky_mail AS sneaky_mail FROM ( - SELECT person.id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS properties___random_uuid + SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS properties___random_uuid FROM person WHERE and(equals(person.team_id, 420), ifNull(in(tuple(person.id, person.version), ( - SELECT person.id, max(person.version) AS version + SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 420) GROUP BY person.id @@ -66,12 +390,20 @@ WHERE ifNull(equals(persons.properties___random_uuid, %(hogql_val_2)s), 0) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT DISTINCT properties.sneaky_mail + FROM persons + WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') + LIMIT 100 ' --- # name: TestQuery.test_query_joins_events_e_pdi ' + -- ClickHouse - SELECT e.event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, e__pdi.distinct_id, e__pdi.person_id + SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, e__pdi.distinct_id AS distinct_id, e__pdi.person_id AS person_id FROM events AS e INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -81,12 +413,19 @@ WHERE equals(e.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, e.timestamp, e.pdi.distinct_id, pdi.person_id + FROM events AS e + LIMIT 10 ' --- # name: TestQuery.test_query_joins_events_pdi ' + -- ClickHouse - SELECT events.event, toTimeZone(events.timestamp, %(hogql_val_0)s) AS timestamp, events__pdi.distinct_id, events__pdi.person_id + SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_0)s) AS timestamp, events__pdi.distinct_id AS distinct_id, events__pdi.person_id AS person_id FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -96,22 +435,29 @@ WHERE equals(events.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, timestamp, pdi.distinct_id, pdi.person_id + FROM events + LIMIT 10 ' --- # name: TestQuery.test_query_joins_events_pdi_e_person_properties ' + -- ClickHouse - SELECT e.event, toTimeZone(e.timestamp, %(hogql_val_1)s) AS timestamp, e__pdi.distinct_id, e__pdi__person.properties___sneaky_mail AS sneaky_mail + SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_1)s) AS timestamp, e__pdi.distinct_id AS distinct_id, e__pdi__person.properties___sneaky_mail AS sneaky_mail FROM events AS e INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, 420) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN ( - SELECT person.id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail + SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail FROM person WHERE and(equals(person.team_id, 420), ifNull(in(tuple(person.id, person.version), ( - SELECT person.id, max(person.version) AS version + SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 420) GROUP BY person.id @@ -120,12 +466,19 @@ WHERE equals(e.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, e.timestamp, pdi.distinct_id, e.pdi.person.properties.sneaky_mail + FROM events AS e + LIMIT 10 ' --- # name: TestQuery.test_query_joins_events_pdi_person ' + -- ClickHouse - SELECT events.event, toTimeZone(events.timestamp, %(hogql_val_0)s) AS timestamp, events__pdi.distinct_id, events__pdi__person.id + SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_0)s) AS timestamp, events__pdi.distinct_id AS distinct_id, events__pdi__person.id AS id FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -141,22 +494,29 @@ WHERE equals(events.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, timestamp, pdi.distinct_id, pdi.person.id + FROM events + LIMIT 10 ' --- # name: TestQuery.test_query_joins_events_pdi_person_properties ' + -- ClickHouse - SELECT events.event, toTimeZone(events.timestamp, %(hogql_val_1)s) AS timestamp, events__pdi.distinct_id, events__pdi__person.properties___sneaky_mail AS sneaky_mail + SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_1)s) AS timestamp, events__pdi.distinct_id AS distinct_id, events__pdi__person.properties___sneaky_mail AS sneaky_mail FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, 420) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) INNER JOIN ( - SELECT person.id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail + SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail FROM person WHERE and(equals(person.team_id, 420), ifNull(in(tuple(person.id, person.version), ( - SELECT person.id, max(person.version) AS version + SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 420) GROUP BY person.id @@ -165,22 +525,29 @@ WHERE equals(events.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, timestamp, pdi.distinct_id, pdi.person.properties.sneaky_mail + FROM events + LIMIT 10 ' --- # name: TestQuery.test_query_joins_events_person_properties ' + -- ClickHouse - SELECT e.event, toTimeZone(e.timestamp, %(hogql_val_1)s) AS timestamp, e__pdi__person.properties___sneaky_mail AS sneaky_mail + SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_1)s) AS timestamp, e__pdi__person.properties___sneaky_mail AS sneaky_mail FROM events AS e INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, 420) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN ( - SELECT person.id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail + SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail FROM person WHERE and(equals(person.team_id, 420), ifNull(in(tuple(person.id, person.version), ( - SELECT person.id, max(person.version) AS version + SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 420) GROUP BY person.id @@ -189,10 +556,17 @@ WHERE equals(e.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, e.timestamp, e.pdi.person.properties.sneaky_mail + FROM events AS e + LIMIT 10 ' --- # name: TestQuery.test_query_joins_events_person_properties_in_aggregration ' + -- ClickHouse SELECT s__pdi__person.properties___sneaky_mail AS sneaky_mail, count() FROM events AS s INNER JOIN ( @@ -201,10 +575,10 @@ WHERE equals(person_distinct_id2.team_id, 420) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS s__pdi ON equals(s.distinct_id, s__pdi.distinct_id) INNER JOIN ( - SELECT person.id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail + SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail FROM person WHERE and(equals(person.team_id, 420), ifNull(in(tuple(person.id, person.version), ( - SELECT person.id, max(person.version) AS version + SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 420) GROUP BY person.id @@ -214,14 +588,22 @@ GROUP BY s__pdi__person.properties___sneaky_mail LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT s.pdi.person.properties.sneaky_mail, count() + FROM events AS s + GROUP BY s.pdi.person.properties.sneaky_mail + LIMIT 10 ' --- # name: TestQuery.test_query_joins_pdi ' + -- ClickHouse - SELECT e.event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, pdi.person_id + SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, pdi.person_id AS person_id FROM events AS e INNER JOIN ( - SELECT person_distinct_id2.distinct_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id + SELECT person_distinct_id2.distinct_id AS distinct_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, 420) GROUP BY person_distinct_id2.distinct_id @@ -229,17 +611,28 @@ WHERE equals(e.team_id, 420) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, timestamp, pdi.person_id + FROM events AS e INNER JOIN ( + SELECT distinct_id, argMax(person_id, version) AS person_id + FROM raw_person_distinct_ids + GROUP BY distinct_id + HAVING equals(argMax(is_deleted, version), 0)) AS pdi ON equals(e.distinct_id, pdi.distinct_id) + LIMIT 100 ' --- # name: TestQuery.test_query_joins_pdi_person_properties ' + -- ClickHouse - SELECT pdi.distinct_id, pdi__person.properties___sneaky_mail AS sneaky_mail + SELECT pdi.distinct_id AS distinct_id, pdi__person.properties___sneaky_mail AS sneaky_mail FROM person_distinct_id2 AS pdi INNER JOIN ( - SELECT person.id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail + SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail FROM person WHERE and(equals(person.team_id, 420), ifNull(in(tuple(person.id, person.version), ( - SELECT person.id, max(person.version) AS version + SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 420) GROUP BY person.id @@ -248,12 +641,19 @@ WHERE equals(pdi.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT pdi.distinct_id, pdi.person.properties.sneaky_mail + FROM person_distinct_ids AS pdi + LIMIT 10 ' --- # name: TestQuery.test_query_joins_pdi_persons ' + -- ClickHouse - SELECT pdi.distinct_id, toTimeZone(pdi__person.created_at, %(hogql_val_0)s) AS created_at + SELECT pdi.distinct_id AS distinct_id, toTimeZone(pdi__person.created_at, %(hogql_val_0)s) AS created_at FROM person_distinct_id2 AS pdi INNER JOIN ( SELECT argMax(person.created_at, person.version) AS created_at, person.id AS id FROM person @@ -264,22 +664,36 @@ WHERE equals(pdi.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT pdi.distinct_id, pdi.person.created_at + FROM person_distinct_ids AS pdi + LIMIT 10 ' --- # name: TestQuery.test_query_joins_simple ' + -- ClickHouse - SELECT e.event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, pdi.distinct_id, p.id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(p.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS sneaky_mail + SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, pdi.distinct_id AS distinct_id, p.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(p.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS sneaky_mail FROM events AS e LEFT JOIN person_distinct_id2 AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN person AS p ON equals(p.id, pdi.person_id) WHERE and(equals(p.team_id, 420), equals(pdi.team_id, 420), equals(e.team_id, 420)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, timestamp, pdi.distinct_id, p.id, p.properties.sneaky_mail + FROM events AS e LEFT JOIN person_distinct_ids AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN persons AS p ON equals(p.id, pdi.person_id) + LIMIT 100 ' --- # name: TestQuery.test_query_person_distinct_ids ' + -- ClickHouse - SELECT DISTINCT person_distinct_ids.person_id, person_distinct_ids.distinct_id + SELECT DISTINCT person_distinct_ids.person_id AS person_id, person_distinct_ids.distinct_id AS distinct_id FROM ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -288,22 +702,29 @@ HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS person_distinct_ids LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT DISTINCT person_id, distinct_id + FROM person_distinct_ids + LIMIT 100 ' --- # name: TestQuery.test_query_select_person_with_joins_without_poe ' + -- ClickHouse - SELECT events.event, toTimeZone(events.timestamp, %(hogql_val_1)s) AS timestamp, events__pdi__person.id, events__pdi__person.properties___sneaky_mail AS sneaky_mail + SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_1)s) AS timestamp, events__pdi__person.id AS id, events__pdi__person.properties___sneaky_mail AS sneaky_mail FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, 420) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) INNER JOIN ( - SELECT person.id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail + SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail FROM person WHERE and(equals(person.team_id, 420), ifNull(in(tuple(person.id, person.version), ( - SELECT person.id, max(person.version) AS version + SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 420) GROUP BY person.id @@ -312,20 +733,34 @@ WHERE equals(events.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, timestamp, person.id, person.properties.sneaky_mail + FROM events + LIMIT 10 ' --- # name: TestQuery.test_query_select_person_with_poe_without_joins ' + -- ClickHouse - SELECT events.event, toTimeZone(events.timestamp, %(hogql_val_0)s) AS timestamp, events.person_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.person_properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS sneaky_mail + SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_0)s) AS timestamp, events.person_id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.person_properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS sneaky_mail FROM events WHERE equals(events.team_id, 420) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event, timestamp, person.id, person.properties.sneaky_mail + FROM events + LIMIT 10 ' --- # name: TestQuery.test_select_person_on_events ' + -- ClickHouse SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(s.person_properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS sneaky_mail, count() FROM events AS s @@ -333,42 +768,74 @@ GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(s.person_properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT poe.properties.sneaky_mail, count() + FROM events AS s + GROUP BY poe.properties.sneaky_mail + LIMIT 10 ' --- # name: TestQuery.test_subquery ' + -- ClickHouse - SELECT count, event + SELECT count AS count, event AS event FROM ( - SELECT count() AS count, events.event + SELECT count() AS count, events.event AS event FROM events WHERE and(equals(events.team_id, 420), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), 0)) GROUP BY events.event) GROUP BY count, event LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT count, event + FROM ( + SELECT count() AS count, event + FROM events + WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') + GROUP BY event) + GROUP BY count, event + LIMIT 100 ' --- # name: TestQuery.test_subquery_alias ' + -- ClickHouse - SELECT c.count, c.event + SELECT c.count AS count, c.event AS event FROM ( - SELECT count(*) AS count, events.event + SELECT count(*) AS count, events.event AS event FROM events WHERE and(equals(events.team_id, 420), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), 0)) GROUP BY events.event) AS c GROUP BY c.count, c.event LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT count, event + FROM ( + SELECT count(*) AS count, event + FROM events + WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') + GROUP BY event) AS c + GROUP BY count, event + LIMIT 100 ' --- # name: TestQuery.test_tuple_access ' + -- ClickHouse - SELECT col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r + SELECT col_a AS col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r FROM ( - SELECT col_a, groupArray(tuple(col_b, col_c)) AS g + SELECT col_a AS col_a, groupArray(tuple(col_b, col_c)) AS g FROM ( SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS col_a, events.event AS col_b, count() AS col_c FROM events @@ -378,16 +845,30 @@ GROUP BY col_a ORDER BY col_a ASC LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r + FROM ( + SELECT col_a, groupArray(tuple(col_b, col_c)) AS g + FROM ( + SELECT properties.index AS col_a, event AS col_b, count() AS col_c + FROM events + GROUP BY properties.index, event) + GROUP BY col_a) + GROUP BY col_a ORDER BY col_a ASC + LIMIT 100 ' --- # name: TestQuery.test_with_pivot_table_1_level ' + -- ClickHouse - SELECT PIVOT_FUNCTION_2.col_a, PIVOT_FUNCTION_2.r + SELECT PIVOT_FUNCTION_2.col_a AS col_a, PIVOT_FUNCTION_2.r AS r FROM ( - SELECT PIVOT_FUNCTION_1.col_a, arrayZip((sumMap((PIVOT_FUNCTION_1.g).1, (PIVOT_FUNCTION_1.g).2) AS x).1, x.2) AS r + SELECT PIVOT_FUNCTION_1.col_a AS col_a, arrayZip((sumMap((PIVOT_FUNCTION_1.g).1, (PIVOT_FUNCTION_1.g).2) AS x).1, x.2) AS r FROM ( - SELECT PIVOT_TABLE_COL_ABC.col_a, groupArray(tuple(PIVOT_TABLE_COL_ABC.col_b, PIVOT_TABLE_COL_ABC.col_c)) AS g + SELECT PIVOT_TABLE_COL_ABC.col_a AS col_a, groupArray(tuple(PIVOT_TABLE_COL_ABC.col_b, PIVOT_TABLE_COL_ABC.col_c)) AS g FROM ( SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS col_a, events.event AS col_b, count() AS col_c FROM events @@ -397,18 +878,34 @@ GROUP BY PIVOT_FUNCTION_1.col_a) AS PIVOT_FUNCTION_2 ORDER BY PIVOT_FUNCTION_2.col_a ASC LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT col_a, r + FROM ( + SELECT col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r + FROM ( + SELECT col_a, groupArray(tuple(col_b, col_c)) AS g + FROM ( + SELECT properties.index AS col_a, event AS col_b, count() AS col_c + FROM events + GROUP BY properties.index, event) AS PIVOT_TABLE_COL_ABC + GROUP BY col_a) AS PIVOT_FUNCTION_1 + GROUP BY col_a) AS PIVOT_FUNCTION_2 ORDER BY col_a ASC + LIMIT 100 ' --- # name: TestQuery.test_with_pivot_table_2_levels ' + -- ClickHouse - SELECT final.col_a, final.r + SELECT final.col_a AS col_a, final.r AS r FROM ( - SELECT PIVOT_FUNCTION_2.col_a, PIVOT_FUNCTION_2.r + SELECT PIVOT_FUNCTION_2.col_a AS col_a, PIVOT_FUNCTION_2.r AS r FROM ( - SELECT PIVOT_FUNCTION_1.col_a, arrayZip((sumMap((PIVOT_FUNCTION_1.g).1, (PIVOT_FUNCTION_1.g).2) AS x).1, x.2) AS r + SELECT PIVOT_FUNCTION_1.col_a AS col_a, arrayZip((sumMap((PIVOT_FUNCTION_1.g).1, (PIVOT_FUNCTION_1.g).2) AS x).1, x.2) AS r FROM ( - SELECT PIVOT_TABLE_COL_ABC.col_a, groupArray(tuple(PIVOT_TABLE_COL_ABC.col_b, PIVOT_TABLE_COL_ABC.col_c)) AS g + SELECT PIVOT_TABLE_COL_ABC.col_a AS col_a, groupArray(tuple(PIVOT_TABLE_COL_ABC.col_b, PIVOT_TABLE_COL_ABC.col_c)) AS g FROM ( SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS col_a, events.event AS col_b, count() AS col_c FROM events @@ -418,5 +915,22 @@ GROUP BY PIVOT_FUNCTION_1.col_a) AS PIVOT_FUNCTION_2) AS final ORDER BY final.col_a ASC LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT col_a, r + FROM ( + SELECT col_a, r + FROM ( + SELECT col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r + FROM ( + SELECT col_a, groupArray(tuple(col_b, col_c)) AS g + FROM ( + SELECT properties.index AS col_a, event AS col_b, count() AS col_c + FROM events + GROUP BY properties.index, event) AS PIVOT_TABLE_COL_ABC + GROUP BY col_a) AS PIVOT_FUNCTION_1 + GROUP BY col_a) AS PIVOT_FUNCTION_2) AS final ORDER BY col_a ASC + LIMIT 100 ' --- diff --git a/posthog/hogql/test/test_modifiers.py b/posthog/hogql/test/test_modifiers.py index 9e253c30042af..4dab98bde188a 100644 --- a/posthog/hogql/test/test_modifiers.py +++ b/posthog/hogql/test/test_modifiers.py @@ -48,30 +48,30 @@ def test_modifiers_persons_on_events_mode_mapping(self): test_cases = [ ( PersonsOnEventsMode.disabled, - "events.event", - "events__pdi__person.id", - "events__pdi__person.properties", + "events.event AS event", + "events__pdi__person.id AS id", + "events__pdi__person.properties AS properties", "toTimeZone(events__pdi__person.created_at, %(hogql_val_0)s) AS created_at", ), ( PersonsOnEventsMode.v1_enabled, - "events.event", - "events.person_id", - "events.person_properties", + "events.event AS event", + "events.person_id AS id", + "events.person_properties AS properties", "toTimeZone(events.person_created_at, %(hogql_val_0)s) AS created_at", ), ( PersonsOnEventsMode.v1_mixed, - "events.event", - "events__pdi.person_id", - "events.person_properties", + "events.event AS event", + "events__pdi.person_id AS id", + "events.person_properties AS properties", "toTimeZone(events__pdi__person.created_at, %(hogql_val_0)s) AS created_at", ), ( PersonsOnEventsMode.v2_enabled, - "events.event", + "events.event AS event", "ifNull(nullIf(events__override.override_person_id, %(hogql_val_0)s), events.person_id) AS id", - "events.person_properties", + "events.person_properties AS properties", "toTimeZone(events.person_created_at, %(hogql_val_1)s) AS created_at", ), ] diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index aad813fd8b373..b752b6cc937a0 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -613,93 +613,93 @@ def test_select_prewhere(self): def test_select_order_by(self): self.assertEqual( self._select("select event from events order by event"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) ORDER BY events.event ASC LIMIT 10000", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) ORDER BY events.event ASC LIMIT 10000", ) self.assertEqual( self._select("select event from events order by event desc"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) ORDER BY events.event DESC LIMIT 10000", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) ORDER BY events.event DESC LIMIT 10000", ) self.assertEqual( self._select("select event from events order by event desc, timestamp"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) ORDER BY events.event DESC, toTimeZone(events.timestamp, %(hogql_val_0)s) ASC LIMIT 10000", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) ORDER BY events.event DESC, toTimeZone(events.timestamp, %(hogql_val_0)s) ASC LIMIT 10000", ) def test_select_limit(self): self.assertEqual( self._select("select event from events limit 10"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", ) self.assertEqual( self._select("select event from events limit 1000000"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", ) self.assertEqual( self._select("select event from events limit (select 100000000)"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT min2(10000, (SELECT 100000000))", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT min2(10000, (SELECT 100000000))", ) self.assertEqual( self._select("select event from events limit (select 100000000) with ties"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT min2(10000, (SELECT 100000000)) WITH TIES", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT min2(10000, (SELECT 100000000)) WITH TIES", ) def test_select_offset(self): # Only the default limit if OFFSET is specified alone self.assertEqual( self._select("select event from events offset 10"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000 OFFSET 10", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000 OFFSET 10", ) self.assertEqual( self._select("select event from events limit 10 offset 10"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 OFFSET 10", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 OFFSET 10", ) self.assertEqual( self._select("select event from events limit 10 offset 0"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 OFFSET 0", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 OFFSET 0", ) self.assertEqual( self._select("select event from events limit 10 with ties offset 0"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 WITH TIES OFFSET 0", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 WITH TIES OFFSET 0", ) def test_select_limit_by(self): self.assertEqual( self._select("select event from events limit 10 offset 0 by 1,event"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 OFFSET 0 BY 1, events.event", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 OFFSET 0 BY 1, events.event", ) def test_select_group_by(self): self.assertEqual( self._select("select event from events group by event, timestamp"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) GROUP BY events.event, toTimeZone(events.timestamp, %(hogql_val_0)s) LIMIT 10000", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) GROUP BY events.event, toTimeZone(events.timestamp, %(hogql_val_0)s) LIMIT 10000", ) def test_select_distinct(self): self.assertEqual( self._select("select distinct event from events group by event, timestamp"), - f"SELECT DISTINCT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) GROUP BY events.event, toTimeZone(events.timestamp, %(hogql_val_0)s) LIMIT 10000", + f"SELECT DISTINCT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) GROUP BY events.event, toTimeZone(events.timestamp, %(hogql_val_0)s) LIMIT 10000", ) def test_select_subquery(self): self.assertEqual( self._select("SELECT event from (select distinct event from events group by event, timestamp)"), - f"SELECT event FROM (SELECT DISTINCT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) GROUP BY events.event, toTimeZone(events.timestamp, %(hogql_val_0)s)) LIMIT 10000", + f"SELECT event AS event FROM (SELECT DISTINCT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) GROUP BY events.event, toTimeZone(events.timestamp, %(hogql_val_0)s)) LIMIT 10000", ) self.assertEqual( self._select("SELECT event from (select distinct event from events group by event, timestamp) e"), - f"SELECT e.event FROM (SELECT DISTINCT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) GROUP BY events.event, toTimeZone(events.timestamp, %(hogql_val_0)s)) AS e LIMIT 10000", + f"SELECT e.event AS event FROM (SELECT DISTINCT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) GROUP BY events.event, toTimeZone(events.timestamp, %(hogql_val_0)s)) AS e LIMIT 10000", ) def test_select_union_all(self): self.assertEqual( self._select("SELECT events.event FROM events UNION ALL SELECT events.event FROM events WHERE 1 = 2"), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000 UNION ALL SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), 0) LIMIT 10000", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000 UNION ALL SELECT events.event AS event FROM events WHERE and(equals(events.team_id, {self.team.pk}), 0) LIMIT 10000", ) self.assertEqual( self._select( "SELECT events.event FROM events UNION ALL SELECT events.event FROM events WHERE 1 = 2 UNION ALL SELECT events.event FROM events WHERE 1 = 2" ), - f"SELECT events.event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000 UNION ALL SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), 0) LIMIT 10000 UNION ALL SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), 0) LIMIT 10000", + f"SELECT events.event AS event FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000 UNION ALL SELECT events.event AS event FROM events WHERE and(equals(events.team_id, {self.team.pk}), 0) LIMIT 10000 UNION ALL SELECT events.event AS event FROM events WHERE and(equals(events.team_id, {self.team.pk}), 0) LIMIT 10000", ) self.assertEqual( self._select("SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1) UNION ALL SELECT 1"), @@ -717,17 +717,17 @@ def test_select_union_all(self): def test_select_sample(self): self.assertEqual( self._select("SELECT events.event FROM events SAMPLE 1"), - f"SELECT events.event FROM events SAMPLE 1 WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", + f"SELECT events.event AS event FROM events SAMPLE 1 WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", ) self.assertEqual( self._select("SELECT events.event FROM events SAMPLE 0.1 OFFSET 1/10"), - f"SELECT events.event FROM events SAMPLE 0.1 OFFSET 1/10 WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", + f"SELECT events.event AS event FROM events SAMPLE 0.1 OFFSET 1/10 WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", ) self.assertEqual( self._select("SELECT events.event FROM events SAMPLE 2/78 OFFSET 999"), - f"SELECT events.event FROM events SAMPLE 2/78 OFFSET 999 WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", + f"SELECT events.event AS event FROM events SAMPLE 2/78 OFFSET 999 WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", ) with override_settings(PERSON_ON_EVENTS_V2_OVERRIDE=False): @@ -741,13 +741,13 @@ def test_select_sample(self): "SELECT events.event FROM events SAMPLE 2/78 OFFSET 999 JOIN persons ON persons.id=events.person_id", context, ), - f"SELECT events.event FROM events SAMPLE 2/78 OFFSET 999 INNER JOIN (SELECT argMax(person_distinct_id2.person_id, " + f"SELECT events.event AS event FROM events SAMPLE 2/78 OFFSET 999 INNER JOIN (SELECT argMax(person_distinct_id2.person_id, " f"person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 " f"WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING " f"ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi " - f"ON equals(events.distinct_id, events__pdi.distinct_id) JOIN (SELECT person.id FROM person " + f"ON equals(events.distinct_id, events__pdi.distinct_id) JOIN (SELECT person.id AS id FROM person " f"WHERE and(equals(person.team_id, {self.team.pk}), ifNull(in(tuple(person.id, person.version), " - f"(SELECT person.id, max(person.version) AS version FROM person WHERE equals(person.team_id, {self.team.pk}) " + f"(SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, {self.team.pk}) " f"GROUP BY person.id HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) " f"SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, events__pdi.person_id) " f"WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", @@ -763,12 +763,12 @@ def test_select_sample(self): "SELECT events.event FROM events SAMPLE 2/78 OFFSET 999 JOIN persons SAMPLE 0.1 ON persons.id=events.person_id", context, ), - f"SELECT events.event FROM events SAMPLE 2/78 OFFSET 999 INNER JOIN (SELECT argMax(person_distinct_id2.person_id, " + f"SELECT events.event AS event FROM events SAMPLE 2/78 OFFSET 999 INNER JOIN (SELECT argMax(person_distinct_id2.person_id, " f"person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 " f"WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING " f"ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi " - f"ON equals(events.distinct_id, events__pdi.distinct_id) JOIN (SELECT person.id FROM person WHERE " - f"and(equals(person.team_id, {self.team.pk}), ifNull(in(tuple(person.id, person.version), (SELECT person.id, " + f"ON equals(events.distinct_id, events__pdi.distinct_id) JOIN (SELECT person.id AS id FROM person WHERE " + f"and(equals(person.team_id, {self.team.pk}), ifNull(in(tuple(person.id, person.version), (SELECT person.id AS id, " f"max(person.version) AS version FROM person WHERE equals(person.team_id, {self.team.pk}) GROUP BY person.id " f"HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) " f"AS persons SAMPLE 0.1 ON equals(persons.id, events__pdi.person_id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", @@ -786,8 +786,8 @@ def test_select_sample(self): ) self.assertEqual( expected, - f"SELECT events.event FROM events SAMPLE 2/78 OFFSET 999 JOIN (SELECT person.id FROM person WHERE " - f"and(equals(person.team_id, {self.team.pk}), ifNull(in(tuple(person.id, person.version), (SELECT person.id, " + f"SELECT events.event AS event FROM events SAMPLE 2/78 OFFSET 999 JOIN (SELECT person.id AS id FROM person WHERE " + f"and(equals(person.team_id, {self.team.pk}), ifNull(in(tuple(person.id, person.version), (SELECT person.id AS id, " f"max(person.version) AS version FROM person WHERE equals(person.team_id, {self.team.pk}) GROUP BY person.id " f"HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) " f"AS persons ON equals(persons.id, events.person_id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", @@ -804,8 +804,8 @@ def test_select_sample(self): ) self.assertEqual( expected, - f"SELECT events.event FROM events SAMPLE 2/78 OFFSET 999 JOIN (SELECT person.id FROM person WHERE " - f"and(equals(person.team_id, {self.team.pk}), ifNull(in(tuple(person.id, person.version), (SELECT person.id, " + f"SELECT events.event AS event FROM events SAMPLE 2/78 OFFSET 999 JOIN (SELECT person.id AS id FROM person WHERE " + f"and(equals(person.team_id, {self.team.pk}), ifNull(in(tuple(person.id, person.version), (SELECT person.id AS id, " f"max(person.version) AS version FROM person WHERE equals(person.team_id, {self.team.pk}) GROUP BY person.id " f"HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) " f"AS persons SAMPLE 0.1 ON equals(persons.id, events.person_id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10000", @@ -895,7 +895,7 @@ def test_window_functions(self): self._select( "SELECT distinct_id, min(timestamp) over win1 as timestamp FROM events WINDOW win1 as (PARTITION by distinct_id ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)" ), - f"SELECT events.distinct_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) OVER win1 AS timestamp FROM events WHERE equals(events.team_id, {self.team.pk}) WINDOW win1 AS (PARTITION BY events.distinct_id ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) LIMIT 10000", + f"SELECT events.distinct_id AS distinct_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) OVER win1 AS timestamp FROM events WHERE equals(events.team_id, {self.team.pk}) WINDOW win1 AS (PARTITION BY events.distinct_id ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) LIMIT 10000", ) def test_window_functions_with_window(self): @@ -903,7 +903,7 @@ def test_window_functions_with_window(self): self._select( "SELECT distinct_id, min(timestamp) over win1 as timestamp FROM events WINDOW win1 as (PARTITION by distinct_id ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)" ), - f"SELECT events.distinct_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) OVER win1 AS timestamp FROM events WHERE equals(events.team_id, {self.team.pk}) WINDOW win1 AS (PARTITION BY events.distinct_id ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) LIMIT 10000", + f"SELECT events.distinct_id AS distinct_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) OVER win1 AS timestamp FROM events WHERE equals(events.team_id, {self.team.pk}) WINDOW win1 AS (PARTITION BY events.distinct_id ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) LIMIT 10000", ) def test_nullish_concat(self): @@ -966,7 +966,7 @@ def test_functions_expecting_datetime_arg(self): def test_field_nullable_equals(self): generated_sql_statements1 = self._select( "SELECT " - "min_first_timestamp = toStartOfMonth(now()), " + "start_time = toStartOfMonth(now()), " "now() = now(), " "1 = now(), " "now() = 1, " @@ -980,7 +980,7 @@ def test_field_nullable_equals(self): ) generated_sql_statements2 = self._select( "SELECT " - "equals(min_first_timestamp, toStartOfMonth(now())), " + "equals(start_time, toStartOfMonth(now())), " "equals(now(), now()), " "equals(1, now()), " "equals(now(), 1), " @@ -995,10 +995,10 @@ def test_field_nullable_equals(self): assert generated_sql_statements1 == generated_sql_statements2 assert generated_sql_statements1 == ( f"SELECT " - # min_first_timestamp = toStartOfMonth(now()) + # start_time = toStartOfMonth(now()) # (the return of toStartOfMonth() is treated as "potentially nullable" since we yet have full typing support) - f"ifNull(equals(toTimeZone(session_replay_events.min_first_timestamp, %(hogql_val_0)s), toStartOfMonth(now64(6, %(hogql_val_1)s))), " - f"isNull(toTimeZone(session_replay_events.min_first_timestamp, %(hogql_val_0)s)) and isNull(toStartOfMonth(now64(6, %(hogql_val_1)s)))), " + f"ifNull(equals(toTimeZone(session_replay_events.start_time, %(hogql_val_0)s), toStartOfMonth(now64(6, %(hogql_val_1)s))), " + f"isNull(toTimeZone(session_replay_events.start_time, %(hogql_val_0)s)) and isNull(toStartOfMonth(now64(6, %(hogql_val_1)s)))), " # now() = now() (also two nullable fields) f"ifNull(equals(now64(6, %(hogql_val_2)s), now64(6, %(hogql_val_3)s)), isNull(now64(6, %(hogql_val_2)s)) and isNull(now64(6, %(hogql_val_3)s))), " # 1 = now() @@ -1018,27 +1018,27 @@ def test_field_nullable_equals(self): # null = click_count f"isNull(session_replay_events.click_count) " # ... - f"FROM (SELECT session_replay_events.min_first_timestamp AS min_first_timestamp, sum(session_replay_events.click_count) AS click_count, sum(session_replay_events.keypress_count) AS keypress_count FROM session_replay_events WHERE equals(session_replay_events.team_id, {self.team.pk}) GROUP BY session_replay_events.min_first_timestamp) AS session_replay_events LIMIT 10000" + f"FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, sum(session_replay_events.click_count) AS click_count, sum(session_replay_events.keypress_count) AS keypress_count FROM session_replay_events WHERE equals(session_replay_events.team_id, {self.team.pk})) AS session_replay_events LIMIT 10000" ) def test_field_nullable_not_equals(self): generated_sql1 = self._select( - "SELECT min_first_timestamp != toStartOfMonth(now()), now() != now(), 1 != now(), now() != 1, 1 != 1, " + "SELECT start_time != toStartOfMonth(now()), now() != now(), 1 != now(), now() != 1, 1 != 1, " "click_count != 1, 1 != click_count, click_count != keypress_count, click_count != null, null != click_count " "FROM session_replay_events" ) generated_sql2 = self._select( - "SELECT notEquals(min_first_timestamp, toStartOfMonth(now())), notEquals(now(), now()), notEquals(1, now()), notEquals(now(), 1), notEquals(1, 1), " + "SELECT notEquals(start_time, toStartOfMonth(now())), notEquals(now(), now()), notEquals(1, now()), notEquals(now(), 1), notEquals(1, 1), " "notEquals(click_count, 1), notEquals(1, click_count), notEquals(click_count, keypress_count), notEquals(click_count, null), notEquals(null, click_count) " "FROM session_replay_events" ) assert generated_sql1 == generated_sql2 assert generated_sql1 == ( f"SELECT " - # min_first_timestamp = toStartOfMonth(now()) + # start_time = toStartOfMonth(now()) # (the return of toStartOfMonth() is treated as "potentially nullable" since we yet have full typing support) - f"ifNull(notEquals(toTimeZone(session_replay_events.min_first_timestamp, %(hogql_val_0)s), toStartOfMonth(now64(6, %(hogql_val_1)s))), " - f"isNotNull(toTimeZone(session_replay_events.min_first_timestamp, %(hogql_val_0)s)) or isNotNull(toStartOfMonth(now64(6, %(hogql_val_1)s)))), " + f"ifNull(notEquals(toTimeZone(session_replay_events.start_time, %(hogql_val_0)s), toStartOfMonth(now64(6, %(hogql_val_1)s))), " + f"isNotNull(toTimeZone(session_replay_events.start_time, %(hogql_val_0)s)) or isNotNull(toStartOfMonth(now64(6, %(hogql_val_1)s)))), " # now() = now() (also two nullable fields) f"ifNull(notEquals(now64(6, %(hogql_val_2)s), now64(6, %(hogql_val_3)s)), isNotNull(now64(6, %(hogql_val_2)s)) or isNotNull(now64(6, %(hogql_val_3)s))), " # 1 = now() @@ -1058,7 +1058,7 @@ def test_field_nullable_not_equals(self): # null = click_count f"isNotNull(session_replay_events.click_count) " # ... - f"FROM (SELECT session_replay_events.min_first_timestamp AS min_first_timestamp, sum(session_replay_events.click_count) AS click_count, sum(session_replay_events.keypress_count) AS keypress_count FROM session_replay_events WHERE equals(session_replay_events.team_id, {self.team.pk}) GROUP BY session_replay_events.min_first_timestamp) AS session_replay_events LIMIT 10000" + f"FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, sum(session_replay_events.click_count) AS click_count, sum(session_replay_events.keypress_count) AS keypress_count FROM session_replay_events WHERE equals(session_replay_events.team_id, {self.team.pk})) AS session_replay_events LIMIT 10000" ) def test_field_nullable_like(self): @@ -1291,7 +1291,7 @@ def test_print_hidden_aliases_timestamp(self): ) self.assertEqual( printed, - f"SELECT timestamp FROM (SELECT toTimeZone(events.timestamp, %(hogql_val_0)s), " + f"SELECT timestamp AS timestamp FROM (SELECT toTimeZone(events.timestamp, %(hogql_val_0)s), " f"toTimeZone(events.timestamp, %(hogql_val_1)s) AS timestamp FROM events WHERE equals(events.team_id, {self.team.pk})) " f"LIMIT 10000 SETTINGS readonly=2, max_execution_time=10, allow_experimental_object_type=1", ) @@ -1306,7 +1306,7 @@ def test_print_hidden_aliases_column_override(self): ) self.assertEqual( printed, - f"SELECT event FROM (SELECT toTimeZone(events.timestamp, %(hogql_val_0)s) AS event, " + f"SELECT event AS event FROM (SELECT toTimeZone(events.timestamp, %(hogql_val_0)s) AS event, " f"event FROM events WHERE equals(events.team_id, {self.team.pk})) " f"LIMIT 10000 SETTINGS readonly=2, max_execution_time=10, allow_experimental_object_type=1", ) @@ -1329,7 +1329,7 @@ def test_print_hidden_aliases_properties(self): ) self.assertEqual( printed, - f"SELECT `$browser` FROM (SELECT nullIf(nullIf(events.`mat_$browser`, ''), 'null') AS `$browser` " + f"SELECT `$browser` AS `$browser` FROM (SELECT nullIf(nullIf(events.`mat_$browser`, ''), 'null') AS `$browser` " f"FROM events WHERE equals(events.team_id, {self.team.pk})) LIMIT 10000 " f"SETTINGS readonly=2, max_execution_time=10, allow_experimental_object_type=1", ) @@ -1352,7 +1352,7 @@ def test_print_hidden_aliases_double_property(self): ) self.assertEqual( printed, - f"SELECT `$browser` FROM (SELECT nullIf(nullIf(events.`mat_$browser`, ''), 'null'), " + f"SELECT `$browser` AS `$browser` FROM (SELECT nullIf(nullIf(events.`mat_$browser`, ''), 'null'), " f"nullIf(nullIf(events.`mat_$browser`, ''), 'null') AS `$browser` " # only the second one gets the alias f"FROM events WHERE equals(events.team_id, {self.team.pk})) LIMIT 10000 " f"SETTINGS readonly=2, max_execution_time=10, allow_experimental_object_type=1", diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index 7f9d9fe65e99e..3c1969d3198cb 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -11,7 +11,7 @@ from posthog.hogql.errors import SyntaxException, HogQLException from posthog.hogql.property import property_to_expr from posthog.hogql.query import execute_hogql_query -from posthog.hogql.test.utils import pretty_print_in_tests +from posthog.hogql.test.utils import pretty_print_in_tests, pretty_print_response_in_tests from posthog.models import Cohort from posthog.models.cohort.util import recalculate_cohortpeople from posthog.models.utils import UUIDT @@ -33,7 +33,7 @@ class TestQuery(ClickhouseTestMixin, APIBaseTest): maxDiff = None def _create_random_events(self) -> str: - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" _create_person( properties={"sneaky_mail": "tim@posthog.com", "random_uuid": random_uuid}, team=self.team, @@ -65,11 +65,7 @@ def test_query(self): placeholders={"random_uuid": ast.Constant(value=random_uuid)}, team=self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - f"SELECT count(), event FROM events WHERE equals(properties.random_uuid, '{random_uuid}') GROUP BY event LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results, [(2, "random event")]) @pytest.mark.usefixtures("unittest_snapshot") @@ -82,11 +78,7 @@ def test_subquery(self): placeholders={"random_uuid": ast.Constant(value=random_uuid)}, team=self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - f"SELECT count, event FROM (SELECT count() AS count, event FROM events WHERE equals(properties.random_uuid, '{random_uuid}') GROUP BY event) GROUP BY count, event LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results, [(2, "random event")]) @pytest.mark.usefixtures("unittest_snapshot") @@ -99,11 +91,7 @@ def test_subquery_alias(self): placeholders={"random_uuid": ast.Constant(value=random_uuid)}, team=self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - f"SELECT count, event FROM (SELECT count(*) AS count, event FROM events WHERE equals(properties.random_uuid, '{random_uuid}') GROUP BY event) AS c GROUP BY count, event LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results, [(2, "random event")]) @pytest.mark.usefixtures("unittest_snapshot") @@ -116,11 +104,7 @@ def test_query_distinct(self): placeholders={"random_uuid": ast.Constant(value=random_uuid)}, team=self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - f"SELECT DISTINCT properties.sneaky_mail FROM persons WHERE equals(properties.random_uuid, '{random_uuid}') LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results, [("tim@posthog.com",)]) @pytest.mark.usefixtures("unittest_snapshot") @@ -131,11 +115,7 @@ def test_query_person_distinct_ids(self): f"select distinct person_id, distinct_id from person_distinct_ids", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT DISTINCT person_id, distinct_id FROM person_distinct_ids LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertTrue(len(response.results) > 0) def test_query_timings(self): @@ -166,11 +146,7 @@ def test_query_joins_simple(self): """, self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, timestamp, pdi.distinct_id, p.id, p.properties.sneaky_mail FROM events AS e LEFT JOIN person_distinct_ids AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN persons AS p ON equals(p.id, pdi.person_id) LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], "bla") self.assertEqual(response.results[0][4], "tim@posthog.com") @@ -195,11 +171,7 @@ def test_query_joins_pdi(self): self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, timestamp, pdi.person_id FROM events AS e INNER JOIN (SELECT distinct_id, argMax(person_id, version) AS person_id FROM raw_person_distinct_ids GROUP BY distinct_id HAVING equals(argMax(is_deleted, version), 0)) AS pdi ON equals(e.distinct_id, pdi.distinct_id) LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertTrue(len(response.results) > 0) @pytest.mark.usefixtures("unittest_snapshot") @@ -211,11 +183,7 @@ def test_query_joins_events_pdi(self): "SELECT event, timestamp, pdi.distinct_id, pdi.person_id FROM events LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, timestamp, pdi.distinct_id, pdi.person_id FROM events LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], "bla") self.assertEqual(response.results[0][3], UUID("00000000-0000-4000-8000-000000000000")) @@ -233,7 +201,7 @@ def test_query_joins_events_e_pdi(self): response.hogql, "SELECT event, e.timestamp, e.pdi.distinct_id, pdi.person_id FROM events AS e LIMIT 10", ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], "bla") self.assertEqual(response.results[0][3], UUID("00000000-0000-4000-8000-000000000000")) @@ -251,7 +219,7 @@ def test_query_joins_pdi_persons(self): response.hogql, "SELECT pdi.distinct_id, pdi.person.created_at FROM person_distinct_ids AS pdi LIMIT 10", ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "bla") self.assertEqual( response.results[0][1], @@ -271,7 +239,7 @@ def test_query_joins_pdi_person_properties(self): response.hogql, "SELECT pdi.distinct_id, pdi.person.properties.sneaky_mail FROM person_distinct_ids AS pdi LIMIT 10", ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "bla") self.assertEqual(response.results[0][1], "tim@posthog.com") @@ -284,11 +252,7 @@ def test_query_joins_events_pdi_person(self): "SELECT event, timestamp, pdi.distinct_id, pdi.person.id FROM events LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, timestamp, pdi.distinct_id, pdi.person.id FROM events LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], "bla") self.assertEqual(response.results[0][3], UUID("00000000-0000-4000-8000-000000000000")) @@ -303,11 +267,7 @@ def test_query_joins_events_pdi_person_properties(self): "SELECT event, timestamp, pdi.distinct_id, pdi.person.properties.sneaky_mail FROM events LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, timestamp, pdi.distinct_id, pdi.person.properties.sneaky_mail FROM events LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], "bla") self.assertEqual(response.results[0][3], "tim@posthog.com") @@ -321,11 +281,7 @@ def test_query_joins_events_pdi_e_person_properties(self): "SELECT event, e.timestamp, pdi.distinct_id, e.pdi.person.properties.sneaky_mail FROM events e LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, e.timestamp, pdi.distinct_id, e.pdi.person.properties.sneaky_mail FROM events AS e LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], "bla") self.assertEqual(response.results[0][3], "tim@posthog.com") @@ -339,11 +295,7 @@ def test_query_joins_events_person_properties(self): "SELECT event, e.timestamp, e.pdi.person.properties.sneaky_mail FROM events e LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, e.timestamp, e.pdi.person.properties.sneaky_mail FROM events AS e LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], "tim@posthog.com") @@ -355,11 +307,7 @@ def test_query_joins_events_person_properties_in_aggregration(self): "SELECT s.pdi.person.properties.sneaky_mail, count() FROM events s GROUP BY s.pdi.person.properties.sneaky_mail LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT s.pdi.person.properties.sneaky_mail, count() FROM events AS s GROUP BY s.pdi.person.properties.sneaky_mail LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "tim@posthog.com") @pytest.mark.usefixtures("unittest_snapshot") @@ -370,11 +318,7 @@ def test_select_person_on_events(self): "SELECT poe.properties.sneaky_mail, count() FROM events s GROUP BY poe.properties.sneaky_mail LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT poe.properties.sneaky_mail, count() FROM events AS s GROUP BY poe.properties.sneaky_mail LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "tim@posthog.com") @pytest.mark.usefixtures("unittest_snapshot") @@ -387,11 +331,7 @@ def test_query_select_person_with_joins_without_poe(self): "SELECT event, timestamp, person.id, person.properties.sneaky_mail FROM events LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, timestamp, person.id, person.properties.sneaky_mail FROM events LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], UUID("00000000-0000-4000-8000-000000000000")) self.assertEqual(response.results[0][3], "tim@posthog.com") @@ -406,15 +346,12 @@ def test_query_select_person_with_poe_without_joins(self): "SELECT event, timestamp, person.id, person.properties.sneaky_mail FROM events LIMIT 10", self.team, ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot - self.assertEqual( - response.hogql, - "SELECT event, timestamp, person.id, person.properties.sneaky_mail FROM events LIMIT 10", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], UUID("00000000-0000-4000-8000-000000000000")) self.assertEqual(response.results[0][3], "tim@posthog.com") + @pytest.mark.usefixtures("unittest_snapshot") def test_prop_cohort_basic(self): with freeze_time("2020-01-10"): _create_person( @@ -467,10 +404,7 @@ def test_prop_cohort_basic(self): ) }, ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event, count() FROM events INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) WHERE and(equals(events.team_id, {self.team.pk}), ifNull(in(events__pdi.person_id, (SELECT cohortpeople.person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)) GROUP BY events.event LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results, [("$pageview", 2)]) with override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=False): @@ -484,15 +418,10 @@ def test_prop_cohort_basic(self): ) }, ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event, count(*) FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, " - f"(SELECT cohortpeople.person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), " - f"equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, " - f"cohortpeople.version HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0)))) GROUP BY events.event LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results, [("$pageview", 2)]) + @pytest.mark.usefixtures("unittest_snapshot") def test_prop_cohort_static(self): with freeze_time("2020-01-10"): _create_person( @@ -533,11 +462,7 @@ def test_prop_cohort_static(self): }, ) self.assertEqual(response.results, [("$pageview", 1)]) - - self.assertEqual( - response.clickhouse, - f"SELECT events.event, count() FROM events INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) WHERE and(equals(events.team_id, {self.team.pk}), ifNull(in(events__pdi.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk})))), 0)) GROUP BY events.event LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot with override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=False): response = execute_hogql_query( @@ -550,12 +475,10 @@ def test_prop_cohort_static(self): ) }, ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event, count(*) FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) GROUP BY events.event LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(response.results, [("$pageview", 1)]) + @pytest.mark.usefixtures("unittest_snapshot") def test_join_with_property_materialized_session_id(self): with freeze_time("2020-01-10"): _create_person( @@ -586,22 +509,19 @@ def test_join_with_property_materialized_session_id(self): "select e.event, s.session_id from events e left join session_replay_events s on s.session_id = e.properties.$session_id where e.properties.$session_id is not null limit 10", team=self.team, ) - self.assertEqual( - response.clickhouse, - f"SELECT e.event, s.session_id FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, nullIf(nullIf(e.`$session_id`, ''), 'null')) WHERE and(equals(s.team_id, {self.team.pk}), equals(e.team_id, {self.team.pk}), isNotNull(nullIf(nullIf(e.`$session_id`, ''), 'null'))) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + assert pretty_print_in_tests(response.hogql, self.team.pk) == self.snapshot self.assertEqual(response.results, [("$pageview", "111"), ("$pageview", "111")]) response = execute_hogql_query( "select e.event, s.session_id from session_replay_events s left join events e on e.properties.$session_id = s.session_id where e.properties.$session_id is not null limit 10", team=self.team, ) - self.assertEqual( - response.clickhouse, - f"SELECT e.event, s.session_id FROM session_replay_events AS s LEFT JOIN events AS e ON equals(nullIf(nullIf(e.`$session_id`, ''), 'null'), s.session_id) WHERE and(equals(e.team_id, {self.team.pk}), equals(s.team_id, {self.team.pk}), isNotNull(nullIf(nullIf(e.`$session_id`, ''), 'null'))) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + assert pretty_print_in_tests(response.hogql, self.team.pk) == self.snapshot self.assertEqual(response.results, [("$pageview", "111"), ("$pageview", "111")]) + @pytest.mark.usefixtures("unittest_snapshot") def test_join_with_property_not_materialized(self): with freeze_time("2020-01-10"): _create_person( @@ -632,20 +552,16 @@ def test_join_with_property_not_materialized(self): "select e.event, s.session_id from events e left join session_replay_events s on s.session_id = e.properties.$$$session_id where e.properties.$$$session_id is not null limit 10", team=self.team, ) - self.assertEqual( - response.clickhouse, - f"SELECT e.event, s.session_id FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_0)s), ''), 'null'), '^\"|\"$', '')) WHERE and(equals(s.team_id, {self.team.pk}), equals(e.team_id, {self.team.pk}), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_1)s), ''), 'null'), '^\"|\"$', ''))) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + assert pretty_print_in_tests(response.hogql, self.team.pk) == self.snapshot self.assertEqual(response.results, [("$pageview", "111"), ("$pageview", "111")]) response = execute_hogql_query( "select e.event, s.session_id from session_replay_events s left join events e on e.properties.$$$session_id = s.session_id where e.properties.$$$session_id is not null limit 10", team=self.team, ) - self.assertEqual( - response.clickhouse, - f"SELECT e.event, s.session_id FROM session_replay_events AS s LEFT JOIN events AS e ON equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_0)s), ''), 'null'), '^\"|\"$', ''), s.session_id) WHERE and(equals(e.team_id, {self.team.pk}), equals(s.team_id, {self.team.pk}), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, %(hogql_val_1)s), ''), 'null'), '^\"|\"$', ''))) LIMIT 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + assert pretty_print_in_tests(response.hogql, self.team.pk) == self.snapshot self.assertEqual(response.results, [("$pageview", "111"), ("$pageview", "111")]) @pytest.mark.usefixtures("unittest_snapshot") @@ -656,7 +572,7 @@ def test_hogql_lambdas(self): team=self.team, ) self.assertEqual(response.results, [([2, 4, 6], 1)]) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot @pytest.mark.usefixtures("unittest_snapshot") def test_hogql_arrays(self): @@ -667,7 +583,7 @@ def test_hogql_arrays(self): ) # Following SQL tradition, ClickHouse array indexes start at 1, not from zero. self.assertEqual(response.results, [([1, 2, 3], 10)]) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot @pytest.mark.usefixtures("unittest_snapshot") def test_tuple_access(self): @@ -696,7 +612,7 @@ def test_tuple_access(self): response.results, [("0", [("random event", 1)]), ("1", [("random event", 1)])], ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot def test_null_properties(self): with freeze_time("2020-01-10"): @@ -741,7 +657,7 @@ def test_null_properties(self): ) def test_window_functions_simple(self): - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" for person in range(5): distinct_id = f"person_{person}_{random_uuid}" with freeze_time("2020-01-10 00:00:00"): @@ -816,7 +732,7 @@ def test_window_functions_simple(self): self.assertEqual(response.results, expected) def test_window_functions_with_window(self): - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" for person in range(5): distinct_id = f"person_{person}_{random_uuid}" with freeze_time("2020-01-10 00:00:00"): @@ -960,7 +876,7 @@ def test_with_pivot_table_1_level(self): response.results, [("0", [("random event", 1)]), ("1", [("random event", 1)])], ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot @pytest.mark.usefixtures("unittest_snapshot") def test_with_pivot_table_2_levels(self): @@ -999,11 +915,11 @@ def test_with_pivot_table_2_levels(self): response.results, [("0", [("random event", 1)]), ("1", [("random event", 1)])], ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot def test_property_access_with_arrays(self): with freeze_time("2020-01-10"): - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" _create_person(team=self.team, distinct_ids=[f"P{random_uuid}"], is_identified=True) _create_event( distinct_id=f"P{random_uuid}", @@ -1400,6 +1316,7 @@ def test_view_link(self): self.assertEqual(response.results, [("bla",), ("bla",), ("bla",), ("bla",)]) + @pytest.mark.usefixtures("unittest_snapshot") def test_hogql_query_filters(self): with freeze_time("2020-01-10"): random_uuid = self._create_random_events() @@ -1416,26 +1333,14 @@ def test_hogql_query_filters(self): ) placeholders = {"distinct_id": ast.Constant(value=random_uuid)} response = execute_hogql_query(query, team=self.team, filters=filters, placeholders=placeholders) - self.assertEqual( - response.hogql, - f"SELECT event, distinct_id FROM events WHERE and(equals(distinct_id, '{random_uuid}'), equals(properties.index, '4')) LIMIT 100", - ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event, events.distinct_id FROM events WHERE and(equals(events.team_id, {self.team.pk}), equals(events.distinct_id, %(hogql_val_0)s), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^\"|\"$', ''), %(hogql_val_2)s), 0)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + assert pretty_print_in_tests(response.hogql, self.team.pk) == self.snapshot self.assertEqual(len(response.results), 1) filters.dateRange = DateRange(date_from="2020-01-01", date_to="2020-01-02") response = execute_hogql_query(query, team=self.team, filters=filters, placeholders=placeholders) - self.assertEqual( - response.hogql, - f"SELECT event, distinct_id FROM events WHERE and(equals(distinct_id, '{random_uuid}'), and(equals(properties.index, '4'), less(timestamp, toDateTime('2020-01-02 00:00:00.000000')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) LIMIT 100", - ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event, events.distinct_id FROM events WHERE and(equals(events.team_id, {self.team.pk}), equals(events.distinct_id, %(hogql_val_0)s), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^\"|\"$', ''), %(hogql_val_2)s), 0), less(toTimeZone(events.timestamp, %(hogql_val_3)s), toDateTime64('2020-01-02 00:00:00.000000', 6, 'UTC')), greaterOrEquals(toTimeZone(events.timestamp, %(hogql_val_4)s), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')))) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + assert pretty_print_in_tests(response.hogql, self.team.pk) == self.snapshot self.assertEqual(len(response.results), 0) filters.dateRange = DateRange(date_from="2020-01-01", date_to="2020-02-02") @@ -1480,7 +1385,7 @@ def test_hogql_query_filters_alias(self): response.hogql, f"SELECT event, distinct_id FROM events AS e WHERE equals(properties.random_uuid, '{random_uuid}') LIMIT 100", ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(len(response.results), 2) @pytest.mark.usefixtures("unittest_snapshot") @@ -1491,7 +1396,7 @@ def test_hogql_union_all_limits(self): response.hogql, f"SELECT event FROM events LIMIT 100 UNION ALL SELECT event FROM events LIMIT 100", ) - assert pretty_print_in_tests(response.clickhouse, self.team.pk) == self.snapshot + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot def test_events_sessions_table(self): with freeze_time("2020-01-10 12:00:00"): diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index b7afa076bb362..266a8cdeb65cc 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -350,9 +350,9 @@ def test_visit_hogqlx_tag_source(self): hogql = print_prepared_ast(node, HogQLContext(team_id=self.team.pk, enable_select_queries=True), "hogql") expected = ( f"SELECT id, email FROM " - f"(SELECT id, properties.email AS email FROM persons WHERE in(id, " - f"(SELECT DISTINCT person_id FROM events)" - f") ORDER BY id ASC LIMIT 101 OFFSET 0) " + f"(SELECT id, properties.email AS email FROM persons INNER JOIN " + f"(SELECT DISTINCT person_id FROM events) " + f"AS source ON equals(persons.id, source.person_id) ORDER BY id ASC) " f"LIMIT 10000" ) assert hogql == expected diff --git a/posthog/hogql/test/utils.py b/posthog/hogql/test/utils.py index 7e46c620c997a..c75305274679e 100644 --- a/posthog/hogql/test/utils.py +++ b/posthog/hogql/test/utils.py @@ -1,10 +1,15 @@ import dataclasses import json +import re +from typing import Any + from pydantic import BaseModel def pretty_print_in_tests(query: str, team_id: int) -> str: - return ( + return re.sub( + r"RANDOM_TEST_ID::[a-f0-9\-]+", + "RANDOM_TEST_ID::UUID", query.replace("SELECT", "\nSELECT") .replace("FROM", "\nFROM") .replace("WHERE", "\nWHERE") @@ -12,10 +17,17 @@ def pretty_print_in_tests(query: str, team_id: int) -> str: .replace("HAVING", "\nHAVING") .replace("LIMIT", "\nLIMIT") .replace("SETTINGS", "\nSETTINGS") - .replace(f"team_id, {team_id})", "team_id, 420)") + .replace(f"team_id, {team_id})", "team_id, 420)"), ) +def pretty_print_response_in_tests(response: Any, team_id: int) -> str: + clickhouse = response.clickhouse + hogql = response.hogql + query = "-- ClickHouse\n" + clickhouse + "\n\n-- HogQL\n" + hogql + return pretty_print_in_tests(query, team_id) + + def pretty_dataclasses(obj, seen=None, indent=0): if seen is None: seen = set() diff --git a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr new file mode 100644 index 0000000000000..cc92012f1806f --- /dev/null +++ b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr @@ -0,0 +1,76 @@ +# name: TestInCohort.test_in_cohort_dynamic + ' + -- ClickHouse + + SELECT events.event AS event + FROM events LEFT JOIN ( + SELECT cohortpeople.person_id AS person_id, 1 AS matched + FROM cohortpeople + WHERE and(equals(cohortpeople.team_id, 420), equals(cohortpeople.cohort_id, 1)) + GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version + HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0)) AS in_cohort__1 ON equals(in_cohort__1.person_id, events.person_id) + WHERE and(equals(events.team_id, 420), ifNull(equals(in_cohort__1.matched, 1), 0), equals(events.event, %(hogql_val_0)s)) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events LEFT JOIN ( + SELECT person_id, 1 AS matched + FROM raw_cohort_people + WHERE equals(cohort_id, 1) + GROUP BY person_id, cohort_id, version + HAVING greater(sum(sign), 0)) AS in_cohort__1 ON equals(in_cohort__1.person_id, person_id) + WHERE and(equals(in_cohort__1.matched, 1), equals(event, 'RANDOM_TEST_ID::UUID')) + LIMIT 100 + ' +--- +# name: TestInCohort.test_in_cohort_static + ' + -- ClickHouse + + SELECT events.event AS event + FROM events LEFT JOIN ( + SELECT person_static_cohort.person_id AS person_id, 1 AS matched + FROM person_static_cohort + WHERE and(equals(person_static_cohort.team_id, 420), equals(person_static_cohort.cohort_id, 2))) AS in_cohort__2 ON equals(in_cohort__2.person_id, events.person_id) + WHERE and(equals(events.team_id, 420), ifNull(equals(in_cohort__2.matched, 1), 0)) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events LEFT JOIN ( + SELECT person_id, 1 AS matched + FROM static_cohort_people + WHERE equals(cohort_id, 2)) AS in_cohort__2 ON equals(in_cohort__2.person_id, person_id) + WHERE equals(in_cohort__2.matched, 1) + LIMIT 100 + ' +--- +# name: TestInCohort.test_in_cohort_strings + ' + -- ClickHouse + + SELECT events.event AS event + FROM events LEFT JOIN ( + SELECT person_static_cohort.person_id AS person_id, 1 AS matched + FROM person_static_cohort + WHERE and(equals(person_static_cohort.team_id, 420), equals(person_static_cohort.cohort_id, 3))) AS in_cohort__3 ON equals(in_cohort__3.person_id, events.person_id) + WHERE and(equals(events.team_id, 420), ifNull(equals(in_cohort__3.matched, 1), 0)) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events LEFT JOIN ( + SELECT person_id, 1 AS matched + FROM static_cohort_people + WHERE equals(cohort_id, 3)) AS in_cohort__3 ON equals(in_cohort__3.person_id, person_id) + WHERE equals(in_cohort__3.matched, 1) + LIMIT 100 + ' +--- diff --git a/posthog/hogql/transforms/test/__snapshots__/test_lazy_tables.ambr b/posthog/hogql/transforms/test/__snapshots__/test_lazy_tables.ambr index 4afb16d85f3b5..a0c2eda465c8d 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_lazy_tables.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_lazy_tables.ambr @@ -1,7 +1,7 @@ # name: TestLazyJoins.test_resolve_lazy_table_as_select_table ' - SELECT persons.id, persons.properties___email AS email, persons.`properties___$browser` AS `$browser` + SELECT persons.id AS id, persons.properties___email AS email, persons.`properties___$browser` AS `$browser` FROM ( SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), person.version) AS properties___email, argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), person.version) AS `properties___$browser`, person.id AS id FROM person @@ -15,7 +15,7 @@ # name: TestLazyJoins.test_resolve_lazy_table_as_table_in_join ' - SELECT events.event, events.distinct_id, events__pdi.person_id, persons.properties___email AS email + SELECT events.event AS event, events.distinct_id AS distinct_id, events__pdi.person_id AS person_id, persons.properties___email AS email FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -35,7 +35,7 @@ # name: TestLazyJoins.test_resolve_lazy_tables ' - SELECT events.event, events__pdi.person_id + SELECT events.event AS event, events__pdi.person_id AS person_id FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -87,7 +87,7 @@ # name: TestLazyJoins.test_resolve_lazy_tables_traversed_fields ' - SELECT events.event, events__pdi.person_id + SELECT events.event AS event, events__pdi.person_id AS person_id FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -101,7 +101,7 @@ # name: TestLazyJoins.test_resolve_lazy_tables_two_levels ' - SELECT events.event, events__pdi__person.id + SELECT events.event AS event, events__pdi__person.id AS id FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -121,7 +121,7 @@ # name: TestLazyJoins.test_resolve_lazy_tables_two_levels_properties ' - SELECT events.event, events__pdi__person.`properties___$browser` AS `$browser` + SELECT events.event AS event, events__pdi__person.`properties___$browser` AS `$browser` FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -141,7 +141,7 @@ # name: TestLazyJoins.test_resolve_lazy_tables_two_levels_properties_duplicate ' - SELECT events.event, events__pdi__person.properties, events__pdi__person.properties___name AS name + SELECT events.event AS event, events__pdi__person.properties AS properties, events__pdi__person.properties___name AS name FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 @@ -161,7 +161,7 @@ # name: TestLazyJoins.test_resolve_lazy_tables_two_levels_traversed ' - SELECT events.event, events__pdi__person.id + SELECT events.event AS event, events__pdi__person.id AS id FROM events INNER JOIN ( SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id FROM person_distinct_id2 diff --git a/posthog/hogql/transforms/test/test_in_cohort.py b/posthog/hogql/transforms/test/test_in_cohort.py index 2a6e11aa7a877..5563ab3eda7e6 100644 --- a/posthog/hogql/transforms/test/test_in_cohort.py +++ b/posthog/hogql/transforms/test/test_in_cohort.py @@ -1,9 +1,11 @@ +import pytest from django.test import override_settings from posthog.hogql import ast from posthog.hogql.errors import HogQLException from posthog.hogql.parser import parse_expr from posthog.hogql.query import execute_hogql_query +from posthog.hogql.test.utils import pretty_print_response_in_tests from posthog.models import Cohort from posthog.models.cohort.util import recalculate_cohortpeople from posthog.models.utils import UUIDT @@ -23,7 +25,7 @@ class TestInCohort(BaseTest): maxDiff = None def _create_random_events(self) -> str: - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" _create_person( properties={"$os": "Chrome", "random_uuid": random_uuid}, team=self.team, @@ -34,6 +36,7 @@ def _create_random_events(self) -> str: flush_persons_and_events() return random_uuid + @pytest.mark.usefixtures("unittest_snapshot") @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=False) def test_in_cohort_dynamic(self): random_uuid = self._create_random_events() @@ -47,17 +50,11 @@ def test_in_cohort_dynamic(self): self.team, modifiers=HogQLQueryModifiers(inCohortVia="leftjoin"), ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event FROM events LEFT JOIN (SELECT cohortpeople.person_id, 1 AS matched FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0)) AS in_cohort__{cohort.pk} ON equals(in_cohort__{cohort.pk}.person_id, events.person_id) WHERE and(equals(events.team_id, {self.team.pk}), ifNull(equals(in_cohort__{cohort.pk}.matched, 1), 0), equals(events.event, %(hogql_val_0)s)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) - self.assertEqual( - response.hogql, - f"SELECT event FROM events LEFT JOIN (SELECT person_id, 1 AS matched FROM raw_cohort_people WHERE equals(cohort_id, {cohort.pk}) GROUP BY person_id, cohort_id, version HAVING greater(sum(sign), 0)) AS in_cohort__{cohort.pk} ON equals(in_cohort__{cohort.pk}.person_id, person_id) WHERE and(equals(in_cohort__{cohort.pk}.matched, 1), equals(event, '{random_uuid}')) LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot self.assertEqual(len(response.results), 1) self.assertEqual(response.results[0][0], random_uuid) + @pytest.mark.usefixtures("unittest_snapshot") @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=False) def test_in_cohort_static(self): cohort = Cohort.objects.create( @@ -69,18 +66,12 @@ def test_in_cohort_static(self): self.team, modifiers=HogQLQueryModifiers(inCohortVia="leftjoin"), ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event FROM events LEFT JOIN (SELECT person_static_cohort.person_id, 1 AS matched FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))) AS in_cohort__{cohort.pk} ON equals(in_cohort__{cohort.pk}.person_id, events.person_id) WHERE and(equals(events.team_id, {self.team.pk}), ifNull(equals(in_cohort__{cohort.pk}.matched, 1), 0)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) - self.assertEqual( - response.hogql, - f"SELECT event FROM events LEFT JOIN (SELECT person_id, 1 AS matched FROM static_cohort_people WHERE equals(cohort_id, {cohort.pk})) AS in_cohort__{cohort.pk} ON equals(in_cohort__{cohort.pk}.person_id, person_id) WHERE equals(in_cohort__{cohort.pk}.matched, 1) LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + @pytest.mark.usefixtures("unittest_snapshot") @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=False) def test_in_cohort_strings(self): - cohort = Cohort.objects.create( + Cohort.objects.create( team=self.team, name="my cohort", is_static=True, @@ -90,15 +81,9 @@ def test_in_cohort_strings(self): self.team, modifiers=HogQLQueryModifiers(inCohortVia="leftjoin"), ) - self.assertEqual( - response.clickhouse, - f"SELECT events.event FROM events LEFT JOIN (SELECT person_static_cohort.person_id, 1 AS matched FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))) AS in_cohort__{cohort.pk} ON equals(in_cohort__{cohort.pk}.person_id, events.person_id) WHERE and(equals(events.team_id, {self.team.pk}), ifNull(equals(in_cohort__{cohort.pk}.matched, 1), 0)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1", - ) - self.assertEqual( - response.hogql, - f"SELECT event FROM events LEFT JOIN (SELECT person_id, 1 AS matched FROM static_cohort_people WHERE equals(cohort_id, {cohort.pk})) AS in_cohort__{cohort.pk} ON equals(in_cohort__{cohort.pk}.person_id, person_id) WHERE equals(in_cohort__{cohort.pk}.matched, 1) LIMIT 100", - ) + assert pretty_print_response_in_tests(response, self.team.pk) == self.snapshot + @pytest.mark.usefixtures("unittest_snapshot") @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=True) def test_in_cohort_error(self): with self.assertRaises(HogQLException) as e: diff --git a/posthog/hogql_queries/actor_strategies.py b/posthog/hogql_queries/actor_strategies.py new file mode 100644 index 0000000000000..466b52ff18d79 --- /dev/null +++ b/posthog/hogql_queries/actor_strategies.py @@ -0,0 +1,175 @@ +from typing import Dict, List, cast, Literal, Optional + +from django.db.models import Prefetch + +from posthog.hogql import ast +from posthog.hogql.property import property_to_expr +from posthog.hogql_queries.insights.paginators import HogQLHasMorePaginator +from posthog.models import Team, Person, Group +from posthog.schema import PersonsQuery + + +class ActorStrategy: + field: str + origin: str + origin_id: str + + def __init__(self, team: Team, query: PersonsQuery, paginator: HogQLHasMorePaginator): + self.team = team + self.paginator = paginator + self.query = query + + def get_actors(self, actor_ids) -> Dict[str, Dict]: + raise NotImplementedError() + + def input_columns(self) -> List[str]: + raise NotImplementedError() + + def filter_conditions(self) -> List[ast.Expr]: + return [] + + def order_by(self) -> Optional[List[ast.OrderExpr]]: + return None + + +class PersonStrategy(ActorStrategy): + field = "person" + origin = "persons" + origin_id = "id" + + def get_actors(self, actor_ids) -> Dict[str, Dict]: + return { + str(p.uuid): { + "id": p.uuid, + **{field: getattr(p, field) for field in ("distinct_ids", "properties", "created_at", "is_identified")}, + } + for p in Person.objects.filter( + team_id=self.team.pk, persondistinctid__team_id=self.team.pk, uuid__in=actor_ids + ) + .prefetch_related(Prefetch("persondistinctid_set", to_attr="distinct_ids_cache")) + .iterator(chunk_size=self.paginator.limit) + } + + def input_columns(self) -> List[str]: + return ["person", "id", "created_at", "person.$delete"] + + def filter_conditions(self) -> List[ast.Expr]: + where_exprs: List[ast.Expr] = [] + + if self.query.properties: + where_exprs.append(property_to_expr(self.query.properties, self.team, scope="person")) + + if self.query.fixedProperties: + where_exprs.append(property_to_expr(self.query.fixedProperties, self.team, scope="person")) + + if self.query.search is not None and self.query.search != "": + where_exprs.append( + ast.Or( + exprs=[ + ast.CompareOperation( + op=ast.CompareOperationOp.ILike, + left=ast.Field(chain=["properties", "email"]), + right=ast.Constant(value=f"%{self.query.search}%"), + ), + ast.CompareOperation( + op=ast.CompareOperationOp.ILike, + left=ast.Field(chain=["properties", "name"]), + right=ast.Constant(value=f"%{self.query.search}%"), + ), + ast.CompareOperation( + op=ast.CompareOperationOp.ILike, + left=ast.Call(name="toString", args=[ast.Field(chain=["id"])]), + right=ast.Constant(value=f"%{self.query.search}%"), + ), + ast.CompareOperation( + op=ast.CompareOperationOp.ILike, + left=ast.Field(chain=["pdi", "distinct_id"]), + right=ast.Constant(value=f"%{self.query.search}%"), + ), + ] + ) + ) + return where_exprs + + def order_by(self) -> Optional[List[ast.OrderExpr]]: + if self.query.orderBy not in [["person"], ["person DESC"], ["person ASC"]]: + return None + + order_property = ( + "email" if self.team.person_display_name_properties is None else self.team.person_display_name_properties[0] + ) + return [ + ast.OrderExpr( + expr=ast.Field(chain=["properties", order_property]), + order=cast( + Literal["ASC", "DESC"], + "DESC" if self.query.orderBy[0] == "person DESC" else "ASC", + ), + ) + ] + + +class GroupStrategy(ActorStrategy): + field = "group" + origin = "groups" + origin_id = "key" + + def __init__(self, group_type_index: int, **kwargs): + self.group_type_index = group_type_index + super().__init__(**kwargs) + + def get_actors(self, actor_ids) -> Dict[str, Dict]: + return { + str(p["group_key"]): { + "id": p["group_key"], + "type": "group", + "properties": p["group_properties"], # TODO: Legacy for frontend + **p, + } + for p in Group.objects.filter( + team_id=self.team.pk, group_type_index=self.group_type_index, group_key__in=actor_ids + ) + .values("group_key", "group_type_index", "created_at", "group_properties") + .iterator(chunk_size=self.paginator.limit) + } + + def input_columns(self) -> List[str]: + return ["group"] + + def filter_conditions(self) -> List[ast.Expr]: + where_exprs: List[ast.Expr] = [] + + if self.query.search is not None and self.query.search != "": + where_exprs.append( + ast.Or( + exprs=[ + ast.CompareOperation( + op=ast.CompareOperationOp.ILike, + left=ast.Field(chain=["properties", "name"]), + right=ast.Constant(value=f"%{self.query.search}%"), + ), + ast.CompareOperation( + op=ast.CompareOperationOp.ILike, + left=ast.Call(name="toString", args=[ast.Field(chain=["key"])]), + right=ast.Constant(value=f"%{self.query.search}%"), + ), + ] + ) + ) + + return where_exprs + + def order_by(self) -> Optional[List[ast.OrderExpr]]: + if self.query.orderBy not in [["group"], ["group DESC"], ["group ASC"]]: + return None + + order_property = "name" + return [ + ast.OrderExpr( + expr=ast.Field(chain=["properties", order_property]), + order=cast( + Literal["ASC", "DESC"], + "DESC" if self.query.orderBy[0] == "group DESC" else "ASC", + ), + ) + ] diff --git a/posthog/hogql_queries/events_query_runner.py b/posthog/hogql_queries/events_query_runner.py index bc9e9810f3698..df88a1c48d162 100644 --- a/posthog/hogql_queries/events_query_runner.py +++ b/posthog/hogql_queries/events_query_runner.py @@ -10,11 +10,10 @@ from posthog.api.utils import get_pk_or_uuid from posthog.clickhouse.client.connection import Workload from posthog.hogql import ast -from posthog.hogql.constants import get_max_limit_for_context, get_default_limit_for_context from posthog.hogql.parser import parse_expr, parse_order_expr from posthog.hogql.property import action_to_expr, has_aggregation, property_to_expr -from posthog.hogql.query import execute_hogql_query from posthog.hogql.timings import HogQLTimings +from posthog.hogql_queries.insights.paginators import HogQLHasMorePaginator from posthog.hogql_queries.query_runner import QueryRunner from posthog.models import Action, Person from posthog.models.element import chain_to_elements @@ -40,15 +39,18 @@ class EventsQueryRunner(QueryRunner): query: EventsQuery query_type = EventsQuery + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.paginator = HogQLHasMorePaginator.from_limit_context( + limit_context=self.limit_context, limit=self.query.limit, offset=self.query.offset + ) + def to_query(self) -> ast.SelectQuery: # Note: This code is inefficient and problematic, see https://github.com/PostHog/posthog/issues/13485 for details. if self.timings is None: self.timings = HogQLTimings() with self.timings.measure("build_ast"): - # limit & offset - offset = 0 if self.query.offset is None else self.query.offset - # columns & group_by with self.timings.measure("columns"): select_input: List[str] = [] @@ -175,13 +177,11 @@ def to_query(self) -> ast.SelectQuery: having=having, group_by=group_by if has_any_aggregation else None, order_by=order_by, - limit=ast.Constant(value=self.limit()), - offset=ast.Constant(value=offset), ) return stmt def calculate(self) -> EventsQueryResponse: - query_result = execute_hogql_query( + query_result = self.paginator.execute_hogql_query( query=self.to_query(), team=self.team, workload=Workload.ONLINE, @@ -195,8 +195,8 @@ def calculate(self) -> EventsQueryResponse: if "*" in self.select_input_raw(): with self.timings.measure("expand_asterisk"): star_idx = self.select_input_raw().index("*") - for index, result in enumerate(query_result.results): - query_result.results[index] = list(result) + for index, result in enumerate(self.paginator.results): + self.paginator.results[index] = list(result) select = result[star_idx] new_result = dict(zip(SELECT_STAR_FROM_EVENTS_FIELDS, select)) new_result["properties"] = json.loads(new_result["properties"]) @@ -204,18 +204,18 @@ def calculate(self) -> EventsQueryResponse: new_result["elements"] = ElementSerializer( chain_to_elements(new_result["elements_chain"]), many=True ).data - query_result.results[index][star_idx] = new_result + self.paginator.results[index][star_idx] = new_result person_indices: List[int] = [] for index, col in enumerate(self.select_input_raw()): if col.split("--")[0].strip() == "person": person_indices.append(index) - if len(person_indices) > 0 and len(query_result.results) > 0: + if len(person_indices) > 0 and len(self.paginator.results) > 0: with self.timings.measure("person_column_extra_query"): # Make a query into postgres to fetch person person_idx = person_indices[0] - distinct_ids = list(set(event[person_idx] for event in query_result.results)) + distinct_ids = list(set(event[person_idx] for event in self.paginator.results)) persons = get_persons_by_distinct_ids(self.team.pk, distinct_ids) persons = persons.prefetch_related(Prefetch("persondistinctid_set", to_attr="distinct_ids_cache")) distinct_to_person: Dict[str, Person] = {} @@ -226,41 +226,34 @@ def calculate(self) -> EventsQueryResponse: # Loop over all columns in case there is more than one "person" column for column_index in person_indices: - for index, result in enumerate(query_result.results): + for index, result in enumerate(self.paginator.results): distinct_id: str = result[column_index] - query_result.results[index] = list(result) + self.paginator.results[index] = list(result) if distinct_to_person.get(distinct_id): person = distinct_to_person[distinct_id] - query_result.results[index][column_index] = { + self.paginator.results[index][column_index] = { "uuid": person.uuid, "created_at": person.created_at, "properties": person.properties or {}, "distinct_id": distinct_id, } else: - query_result.results[index][column_index] = { + self.paginator.results[index][column_index] = { "distinct_id": distinct_id, } - received_extra_row = len(query_result.results) == self.limit() # limit was +=1'd above return EventsQueryResponse( - results=query_result.results[: self.limit() - 1] if received_extra_row else query_result.results, + results=self.paginator.results, columns=self.select_input_raw(), - types=[type for _, type in query_result.types], - hasMore=received_extra_row, + types=[t for _, t in query_result.types] if query_result.types else None, timings=self.timings.to_list(), hogql=query_result.hogql, + **self.paginator.response_params(), ) def select_input_raw(self) -> List[str]: return ["*"] if len(self.query.select) == 0 else self.query.select - def limit(self) -> int: - # adding +1 to the limit to check if there's a "next page" after the requested results - max_rows = get_max_limit_for_context(self.limit_context) - default_rows = get_default_limit_for_context(self.limit_context) - return min(max_rows, default_rows if self.query.limit is None else self.query.limit) + 1 - def _is_stale(self, cached_result_package): return True diff --git a/posthog/hogql_queries/insights/insight_persons_query_runner.py b/posthog/hogql_queries/insights/insight_persons_query_runner.py index e0681bc5af08a..de14b029cc1ac 100644 --- a/posthog/hogql_queries/insights/insight_persons_query_runner.py +++ b/posthog/hogql_queries/insights/insight_persons_query_runner.py @@ -4,6 +4,7 @@ from posthog.hogql import ast from posthog.hogql.query import execute_hogql_query from posthog.hogql_queries.insights.lifecycle_query_runner import LifecycleQueryRunner +from posthog.hogql_queries.insights.retention_query_runner import RetentionQueryRunner from posthog.hogql_queries.insights.trends.trends_query_runner import TrendsQueryRunner from posthog.hogql_queries.query_runner import QueryRunner, get_query_runner from posthog.models.filters.mixins.utils import cached_property @@ -27,12 +28,22 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: elif isinstance(self.source_runner, TrendsQueryRunner): trends_runner = cast(TrendsQueryRunner, self.source_runner) return trends_runner.to_persons_query() + elif isinstance(self.source_runner, RetentionQueryRunner): + retention_runner = cast(RetentionQueryRunner, self.source_runner) + return retention_runner.to_persons_query(interval=self.query.interval) raise ValueError(f"Cannot convert source query of type {self.query.source.kind} to persons query") def to_persons_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: return self.to_query() + @property + def group_type_index(self) -> int | None: + if not self.source_runner or not isinstance(self.source_runner, RetentionQueryRunner): + return None + + return cast(RetentionQueryRunner, self.source_runner).group_type_index + def calculate(self) -> HogQLQueryResponse: return execute_hogql_query( query_type="InsightPersonsQuery", diff --git a/posthog/hogql_queries/insights/lifecycle_query_runner.py b/posthog/hogql_queries/insights/lifecycle_query_runner.py index 826a33ab41559..92cf4e23704cb 100644 --- a/posthog/hogql_queries/insights/lifecycle_query_runner.py +++ b/posthog/hogql_queries/insights/lifecycle_query_runner.py @@ -115,7 +115,7 @@ def to_persons_query( ) return parse_select( - "SELECT person_id FROM {events_query} WHERE {where}", + "SELECT DISTINCT person_id FROM {events_query} WHERE {where}", placeholders={ "events_query": self.events_query, "where": ast.And(exprs=exprs) if len(exprs) > 0 else ast.Constant(value=1), diff --git a/posthog/hogql_queries/insights/paginators.py b/posthog/hogql_queries/insights/paginators.py new file mode 100644 index 0000000000000..a2fc474346303 --- /dev/null +++ b/posthog/hogql_queries/insights/paginators.py @@ -0,0 +1,77 @@ +from typing import Any, Optional, cast + +from posthog.hogql import ast +from posthog.hogql.constants import ( + get_max_limit_for_context, + get_default_limit_for_context, + LimitContext, + DEFAULT_RETURNED_ROWS, +) +from posthog.hogql.query import execute_hogql_query +from posthog.schema import HogQLQueryResponse + + +class HogQLHasMorePaginator: + """ + Paginator that fetches one more result than requested to determine if there are more results. + Takes care of setting the limit and offset on the query. + """ + + def __init__(self, *, limit: Optional[int], offset: Optional[int]): + self.response: Optional[HogQLQueryResponse] = None + self.results: list[Any] = [] + self.limit = limit if limit and limit > 0 else DEFAULT_RETURNED_ROWS + self.offset = offset if offset and offset > 0 else 0 + + @classmethod + def from_limit_context( + cls, *, limit_context: LimitContext, limit: Optional[int], offset: Optional[int] + ) -> "HogQLHasMorePaginator": + max_rows = get_max_limit_for_context(limit_context) + default_rows = get_default_limit_for_context(limit_context) + limit = min(max_rows, default_rows if (limit is None or limit <= 0) else limit) + return cls(limit=limit, offset=offset) + + def paginate(self, query: ast.SelectQuery) -> ast.SelectQuery: + query.limit = ast.Constant(value=self.limit + 1) + query.offset = ast.Constant(value=self.offset) + return query + + def has_more(self) -> bool: + if not self.response or not self.response.results: + return False + + return len(self.response.results) > self.limit + + def trim_results(self) -> list[Any]: + if not self.response or not self.response.results: + return [] + + if self.has_more(): + return self.response.results[:-1] + + return self.response.results + + def execute_hogql_query( + self, + query_type: str, + query: ast.SelectQuery, + **kwargs, + ) -> HogQLQueryResponse: + self.response = cast( + HogQLQueryResponse, + execute_hogql_query( + query=self.paginate(query), + query_type=query_type, + **kwargs, + ), + ) + self.results = self.trim_results() + return self.response + + def response_params(self): + return { + "hasMore": self.has_more(), + "limit": self.limit, + "offset": self.offset, + } diff --git a/posthog/hogql_queries/insights/retention_query_runner.py b/posthog/hogql_queries/insights/retention_query_runner.py index b1dc46ad3b454..1920d70ee3d5e 100644 --- a/posthog/hogql_queries/insights/retention_query_runner.py +++ b/posthog/hogql_queries/insights/retention_query_runner.py @@ -47,6 +47,10 @@ def __init__( ): super().__init__(query, team=team, timings=timings, modifiers=modifiers, limit_context=limit_context) + @property + def group_type_index(self) -> int | None: + return self.query.aggregation_group_type_index + def get_applicable_entity(self, event_query_type): default_entity = RetentionEntity( **{ @@ -71,8 +75,8 @@ def retention_events_query(self, event_query_type) -> ast.SelectQuery: event_date_expr = start_of_interval_sql target_field = "person_id" - if self.query.aggregation_group_type_index is not None: - group_index = int(self.query.aggregation_group_type_index) + if self.group_type_index is not None: + group_index = int(self.group_type_index) if 0 <= group_index <= 4: target_field = f"$group_{group_index}" @@ -178,12 +182,12 @@ def build_target_event_query(self) -> ast.SelectQuery: def build_returning_event_query(self) -> ast.SelectQuery: return self.retention_events_query(event_query_type=RetentionQueryType.RETURNING) - def actor_query(self) -> ast.SelectQuery: + def actor_query(self, breakdown_values_filter: Optional[Any] = None) -> ast.SelectQuery: placeholders = { **self.query_date_range.to_placeholders(), "returning_event_query": self.build_returning_event_query(), "target_event_query": self.build_target_event_query(), - "breakdown_values_filter": ast.Constant(value=None), + "breakdown_values_filter": ast.Constant(value=breakdown_values_filter), "selected_interval": ast.Constant(value=None), } return parse_select( @@ -324,3 +328,22 @@ def calculate(self) -> RetentionQueryResponse: ] return RetentionQueryResponse(results=results, timings=response.timings, hogql=hogql) + + def to_persons_query(self, interval: Optional[int] = None) -> ast.SelectQuery: + with self.timings.measure("retention_query"): + retention_query = parse_select( + """ + SELECT + actor_id, + arraySort(groupArray(actor_activity.intervals_from_base)) AS appearances + + FROM {actor_query} AS actor_activity + + GROUP BY actor_id + """, + placeholders={ + "actor_query": self.actor_query(breakdown_values_filter=[interval]), + }, + timings=self.timings, + ) + return retention_query diff --git a/posthog/hogql_queries/insights/test/__snapshots__/test_lifecycle_query_runner.ambr b/posthog/hogql_queries/insights/test/__snapshots__/test_lifecycle_query_runner.ambr index 12cf0173a7f97..5954c42aa3b1a 100644 --- a/posthog/hogql_queries/insights/test/__snapshots__/test_lifecycle_query_runner.ambr +++ b/posthog/hogql_queries/insights/test/__snapshots__/test_lifecycle_query_runner.ambr @@ -2,15 +2,15 @@ ' SELECT groupArray(start_of_period) AS date, groupArray(round(multiply(counts, divide(1, 0.1)))) AS total, - status + status AS status FROM (SELECT if(ifNull(equals(status, 'dormant'), 0), negate(sum(counts)), negate(negate(sum(counts)))) AS counts, - start_of_period, - status + start_of_period AS start_of_period, + status AS status FROM (SELECT periods.start_of_period AS start_of_period, 0 AS counts, - sec.status + sec.status AS status FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS start_of_period FROM numbers(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 00:00:00', 6, 'UTC'))), toStartOfDay(plus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'UTC')), toIntervalDay(1))))) AS numbers) AS periods @@ -20,9 +20,9 @@ (SELECT 1) ARRAY JOIN ['new', 'returning', 'resurrecting', 'dormant'] AS status) AS sec ORDER BY sec.status ASC, start_of_period ASC - UNION ALL SELECT start_of_period, + UNION ALL SELECT start_of_period AS start_of_period, count(DISTINCT person_id) AS counts, - status + status AS status FROM (SELECT events__pdi__person.id AS person_id, min(toTimeZone(events__pdi__person.created_at, 'UTC')) AS created_at, @@ -73,15 +73,15 @@ ' SELECT groupArray(start_of_period) AS date, groupArray(counts) AS total, - status + status AS status FROM (SELECT if(ifNull(equals(status, 'dormant'), 0), negate(sum(counts)), negate(negate(sum(counts)))) AS counts, - start_of_period, - status + start_of_period AS start_of_period, + status AS status FROM (SELECT periods.start_of_period AS start_of_period, 0 AS counts, - sec.status + sec.status AS status FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS start_of_period FROM numbers(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 00:00:00', 6, 'UTC'))), toStartOfDay(plus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'UTC')), toIntervalDay(1))))) AS numbers) AS periods @@ -91,9 +91,9 @@ (SELECT 1) ARRAY JOIN ['new', 'returning', 'resurrecting', 'dormant'] AS status) AS sec ORDER BY sec.status ASC, start_of_period ASC - UNION ALL SELECT start_of_period, + UNION ALL SELECT start_of_period AS start_of_period, count(DISTINCT person_id) AS counts, - status + status AS status FROM (SELECT events__pdi__person.id AS person_id, min(toTimeZone(events__pdi__person.created_at, 'UTC')) AS created_at, @@ -144,15 +144,15 @@ ' SELECT groupArray(start_of_period) AS date, groupArray(counts) AS total, - status + status AS status FROM (SELECT if(ifNull(equals(status, 'dormant'), 0), negate(sum(counts)), negate(negate(sum(counts)))) AS counts, - start_of_period, - status + start_of_period AS start_of_period, + status AS status FROM (SELECT periods.start_of_period AS start_of_period, 0 AS counts, - sec.status + sec.status AS status FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'US/Pacific'))), toIntervalDay(numbers.number)) AS start_of_period FROM numbers(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 00:00:00', 6, 'US/Pacific'))), toStartOfDay(plus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'US/Pacific')), toIntervalDay(1))))) AS numbers) AS periods @@ -162,9 +162,9 @@ (SELECT 1) ARRAY JOIN ['new', 'returning', 'resurrecting', 'dormant'] AS status) AS sec ORDER BY sec.status ASC, start_of_period ASC - UNION ALL SELECT start_of_period, + UNION ALL SELECT start_of_period AS start_of_period, count(DISTINCT person_id) AS counts, - status + status AS status FROM (SELECT events__pdi__person.id AS person_id, min(toTimeZone(events__pdi__person.created_at, 'US/Pacific')) AS created_at, diff --git a/posthog/hogql_queries/insights/test/__snapshots__/test_retention_query_runner.ambr b/posthog/hogql_queries/insights/test/__snapshots__/test_retention_query_runner.ambr index bd6e5436ad1d5..62d2f6910ce2b 100644 --- a/posthog/hogql_queries/insights/test/__snapshots__/test_retention_query_runner.ambr +++ b/posthog/hogql_queries/insights/test/__snapshots__/test_retention_query_runner.ambr @@ -4,9 +4,9 @@ actor_activity.intervals_from_base AS intervals_from_base, count(DISTINCT actor_activity.actor_id) AS count FROM - (SELECT DISTINCT breakdown_values, - intervals_from_base, - actor_id + (SELECT DISTINCT breakdown_values AS breakdown_values, + intervals_from_base AS intervals_from_base, + actor_id AS actor_id FROM (SELECT target_event.breakdown_values AS breakdown_values, dateDiff('day', target_event.event_date, returning_event.event_date) AS intervals_from_base, @@ -85,9 +85,9 @@ actor_activity.intervals_from_base AS intervals_from_base, count(DISTINCT actor_activity.actor_id) AS count FROM - (SELECT DISTINCT breakdown_values, - intervals_from_base, - actor_id + (SELECT DISTINCT breakdown_values AS breakdown_values, + intervals_from_base AS intervals_from_base, + actor_id AS actor_id FROM (SELECT target_event.breakdown_values AS breakdown_values, dateDiff('month', target_event.event_date, returning_event.event_date) AS intervals_from_base, @@ -149,9 +149,9 @@ actor_activity.intervals_from_base AS intervals_from_base, count(DISTINCT actor_activity.actor_id) AS count FROM - (SELECT DISTINCT breakdown_values, - intervals_from_base, - actor_id + (SELECT DISTINCT breakdown_values AS breakdown_values, + intervals_from_base AS intervals_from_base, + actor_id AS actor_id FROM (SELECT target_event.breakdown_values AS breakdown_values, dateDiff('day', target_event.event_date, returning_event.event_date) AS intervals_from_base, @@ -216,9 +216,9 @@ actor_activity.intervals_from_base AS intervals_from_base, count(DISTINCT actor_activity.actor_id) AS count FROM - (SELECT DISTINCT breakdown_values, - intervals_from_base, - actor_id + (SELECT DISTINCT breakdown_values AS breakdown_values, + intervals_from_base AS intervals_from_base, + actor_id AS actor_id FROM (SELECT target_event.breakdown_values AS breakdown_values, dateDiff('day', target_event.event_date, returning_event.event_date) AS intervals_from_base, @@ -236,11 +236,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -277,11 +277,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -303,9 +303,9 @@ actor_activity.intervals_from_base AS intervals_from_base, count(DISTINCT actor_activity.actor_id) AS count FROM - (SELECT DISTINCT breakdown_values, - intervals_from_base, - actor_id + (SELECT DISTINCT breakdown_values AS breakdown_values, + intervals_from_base AS intervals_from_base, + actor_id AS actor_id FROM (SELECT target_event.breakdown_values AS breakdown_values, dateDiff('day', target_event.event_date, returning_event.event_date) AS intervals_from_base, @@ -370,9 +370,9 @@ actor_activity.intervals_from_base AS intervals_from_base, count(DISTINCT actor_activity.actor_id) AS count FROM - (SELECT DISTINCT breakdown_values, - intervals_from_base, - actor_id + (SELECT DISTINCT breakdown_values AS breakdown_values, + intervals_from_base AS intervals_from_base, + actor_id AS actor_id FROM (SELECT target_event.breakdown_values AS breakdown_values, dateDiff('day', target_event.event_date, returning_event.event_date) AS intervals_from_base, @@ -437,9 +437,9 @@ actor_activity.intervals_from_base AS intervals_from_base, count(DISTINCT actor_activity.actor_id) AS count FROM - (SELECT DISTINCT breakdown_values, - intervals_from_base, - actor_id + (SELECT DISTINCT breakdown_values AS breakdown_values, + intervals_from_base AS intervals_from_base, + actor_id AS actor_id FROM (SELECT target_event.breakdown_values AS breakdown_values, dateDiff('week', target_event.event_date, returning_event.event_date) AS intervals_from_base, @@ -504,9 +504,9 @@ actor_activity.intervals_from_base AS intervals_from_base, count(DISTINCT actor_activity.actor_id) AS count FROM - (SELECT DISTINCT breakdown_values, - intervals_from_base, - actor_id + (SELECT DISTINCT breakdown_values AS breakdown_values, + intervals_from_base AS intervals_from_base, + actor_id AS actor_id FROM (SELECT target_event.breakdown_values AS breakdown_values, dateDiff('week', target_event.event_date, returning_event.event_date) AS intervals_from_base, diff --git a/posthog/hogql_queries/insights/test/test_lifecycle_query_runner.py b/posthog/hogql_queries/insights/test/test_lifecycle_query_runner.py index d2876e418edfa..5d5bd40d153e2 100644 --- a/posthog/hogql_queries/insights/test/test_lifecycle_query_runner.py +++ b/posthog/hogql_queries/insights/test/test_lifecycle_query_runner.py @@ -39,7 +39,7 @@ class TestLifecycleQueryRunner(ClickhouseTestMixin, APIBaseTest): maxDiff = None def _create_random_events(self) -> str: - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" _create_person( properties={"sneaky_mail": "tim@posthog.com", "random_uuid": random_uuid}, team=self.team, diff --git a/posthog/hogql_queries/insights/test/test_paginators.py b/posthog/hogql_queries/insights/test/test_paginators.py new file mode 100644 index 0000000000000..7954f35daf7f2 --- /dev/null +++ b/posthog/hogql_queries/insights/test/test_paginators.py @@ -0,0 +1,209 @@ +from posthog.hogql.constants import ( + LimitContext, + get_default_limit_for_context, + get_max_limit_for_context, + MAX_SELECT_RETURNED_ROWS, +) +from posthog.hogql.parser import parse_select +from posthog.hogql_queries.insights.paginators import HogQLHasMorePaginator +from posthog.hogql_queries.persons_query_runner import PersonsQueryRunner +from posthog.models.utils import UUIDT +from posthog.schema import ( + PersonsQuery, + PersonPropertyFilter, + PropertyOperator, +) +from posthog.test.base import ( + APIBaseTest, + ClickhouseTestMixin, + _create_person, + flush_persons_and_events, + _create_event, +) + + +class TestHogQLHasMorePaginator(ClickhouseTestMixin, APIBaseTest): + maxDiff = None + random_uuid: str + + def _create_random_persons(self) -> str: + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" + for index in range(10): + _create_person( + properties={ + "email": f"jacob{index}@{random_uuid}.posthog.com", + "name": f"Mr Jacob {random_uuid}", + "random_uuid": random_uuid, + "index": index, + }, + team=self.team, + distinct_ids=[f"id-{random_uuid}-{index}"], + is_identified=True, + ) + _create_event( + distinct_id=f"id-{random_uuid}-{index}", + event=f"clicky-{index}", + team=self.team, + ) + + flush_persons_and_events() + return random_uuid + + def _create_runner(self, query: PersonsQuery) -> PersonsQueryRunner: + return PersonsQueryRunner(team=self.team, query=query) + + def setUp(self): + super().setUp() + self.random_uuid = self._create_random_persons() + + def test_persons_query_limit(self): + runner = self._create_runner( + PersonsQuery(select=["properties.email"], orderBy=["properties.email DESC"], limit=1) + ) + response = runner.calculate() + self.assertEqual(response.results, [[f"jacob9@{self.random_uuid}.posthog.com"]]) + self.assertEqual(response.hasMore, True) + + runner = self._create_runner( + PersonsQuery( + select=["properties.email"], + orderBy=["properties.email DESC"], + limit=1, + offset=2, + ) + ) + response = runner.calculate() + self.assertEqual(response.results, [[f"jacob7@{self.random_uuid}.posthog.com"]]) + self.assertEqual(response.hasMore, True) + + def test_zero_limit(self): + """Test behavior with limit set to zero.""" + runner = self._create_runner(PersonsQuery(select=["properties.email"], limit=0)) + response = runner.calculate() + self.assertEqual(runner.paginator.limit, 100) + self.assertEqual(response.limit, 100) + self.assertEqual(len(response.results), 10) + self.assertFalse(response.hasMore) + + def test_negative_limit(self): + """Test behavior with negative limit value.""" + runner = self._create_runner(PersonsQuery(select=["properties.email"], limit=-1)) + response = runner.calculate() + self.assertEqual(runner.paginator.limit, 100) + self.assertEqual(response.limit, 100) + self.assertEqual(len(response.results), 10) + self.assertFalse(response.hasMore) + + def test_exact_limit_match(self): + """Test when available items equal the limit.""" + runner = self._create_runner(PersonsQuery(select=["properties.email"], limit=10)) + response = runner.calculate() + self.assertEqual(len(response.results), 10) + self.assertFalse(response.hasMore) + + def test_empty_result_set(self): + """Test behavior when query returns no results.""" + runner = self._create_runner( + PersonsQuery( + select=["properties.email"], + limit=10, + properties=[ + PersonPropertyFilter(key="email", value="random", operator=PropertyOperator.exact), + ], + ) + ) + response = runner.calculate() + self.assertEqual(len(response.results), 0) + self.assertFalse(response.hasMore) + + def test_large_offset(self): + """Test behavior with offset larger than the total number of items.""" + self.random_uuid = self._create_random_persons() + runner = self._create_runner(PersonsQuery(select=["properties.email"], limit=5, offset=100)) + response = runner.calculate() + self.assertEqual(len(response.results), 0) + self.assertFalse(response.hasMore) + + def test_offset_plus_limit_exceeding_total(self): + """Test when sum of offset and limit exceeds total items.""" + runner = self._create_runner(PersonsQuery(select=["properties.email"], limit=10, offset=5)) + response = runner.calculate() + self.assertEqual(runner.paginator.offset, 5) + self.assertEqual(len(response.results), 5) + self.assertFalse(response.hasMore) + + def test_response_params_consistency(self): + """Test consistency of response_params method.""" + paginator = HogQLHasMorePaginator(limit=5, offset=10) + paginator.response = paginator.execute_hogql_query( + "test_query", + parse_select("SELECT * FROM persons"), + team=self.team, + ) + params = paginator.response_params() + self.assertEqual(params["limit"], 5) + self.assertEqual(params["offset"], 10) + self.assertEqual(params["hasMore"], paginator.has_more()) + + def test_handle_none_response(self): + """Test handling of None response.""" + paginator = HogQLHasMorePaginator(limit=5, offset=0) + paginator.response = None # Simulate a None response + self.assertEqual(paginator.trim_results(), []) + self.assertFalse(paginator.has_more()) + + def test_limit_context_variations(self): + limit_context = LimitContext.QUERY + + test_cases = [ + { + "limit": 5, + "offset": 10, + "expected_limit": 5, + "expected_offset": 10, + }, + { + "limit": None, + "offset": 10, + "expected_limit": get_default_limit_for_context(limit_context), + "expected_offset": 10, + }, + { + "limit": 0, + "offset": 10, + "expected_limit": get_default_limit_for_context(limit_context), + "expected_offset": 10, + }, + { + "limit": -1, + "offset": 10, + "expected_limit": get_default_limit_for_context(limit_context), + "expected_offset": 10, + }, + { + "limit": MAX_SELECT_RETURNED_ROWS, + "offset": 10, + "expected_limit": get_max_limit_for_context(limit_context), + "expected_offset": 10, + }, + { + "limit": 5, + "offset": None, + "expected_limit": 5, + "expected_offset": 0, + }, + { + "limit": 5, + "offset": -1, + "expected_limit": 5, + "expected_offset": 0, + }, + ] + + for case in test_cases: + with self.subTest(case=case): + paginator = HogQLHasMorePaginator.from_limit_context( + limit_context=limit_context, limit=case["limit"], offset=case["offset"] + ) + self.assertEqual(paginator.limit, case["expected_limit"]) + self.assertEqual(paginator.offset, case["expected_offset"]) diff --git a/posthog/hogql_queries/insights/test/test_retention_query_runner.py b/posthog/hogql_queries/insights/test/test_retention_query_runner.py index 7a6f076e43385..eacf27a5021bf 100644 --- a/posthog/hogql_queries/insights/test/test_retention_query_runner.py +++ b/posthog/hogql_queries/insights/test/test_retention_query_runner.py @@ -1,8 +1,5 @@ -import json import uuid from datetime import datetime -from typing import Any -from unittest import skip from zoneinfo import ZoneInfo from django.test import override_settings @@ -10,13 +7,12 @@ from posthog.constants import ( RETENTION_FIRST_TIME, - RETENTION_TYPE, TREND_FILTER_TYPE_ACTIONS, TREND_FILTER_TYPE_EVENTS, ) from posthog.hogql_queries.insights.retention_query_runner import RetentionQueryRunner +from posthog.hogql_queries.persons_query_runner import PersonsQueryRunner from posthog.models import Action, ActionStep -from posthog.models.filters import RetentionFilter as OldRetentionFilter from posthog.test.base import ( APIBaseTest, ClickhouseTestMixin, @@ -74,8 +70,23 @@ def run_query(self, query): runner = RetentionQueryRunner(team=self.team, query=query) return runner.calculate().model_dump()["results"] - def actors_in_period(self, *args, **kwargs) -> Any: - return args, kwargs + def run_actors_query(self, interval, query): + query["kind"] = "RetentionQuery" + if not query.get("retentionFilter"): + query["retentionFilter"] = {} + runner = PersonsQueryRunner( + team=self.team, + query={ + "select": ["person", "appearances"], + "orderBy": ["length(appearances) DESC", "actor_id"], + "source": { + "kind": "InsightPersonsQuery", + "interval": interval, + "source": query, + }, + }, + ) + return runner.calculate().model_dump()["results"] def test_retention_default(self): _create_person(team_id=self.team.pk, distinct_ids=["person1", "alias1"]) @@ -711,7 +722,6 @@ def test_interval_rounding(self): ], ) - @skip("TODO: Bring back when working on actors_in_period") def test_retention_people_basic(self): person1 = _create_person(team_id=self.team.pk, distinct_ids=["person1", "alias1"]) _create_person(team_id=self.team.pk, distinct_ids=["person2"]) @@ -733,51 +743,44 @@ def test_retention_people_basic(self): ) # even if set to hour 6 it should default to beginning of day and include all pageviews above - result, _ = self.actors_in_period( - OldRetentionFilter( - data={"date_to": _date(10, hour=6), "selected_interval": 0}, - team=self.team, - ), - self.team, + result = self.run_actors_query( + interval=0, + query={ + "dateRange": {"date_to": _date(10, hour=6)}, + }, ) - self.assertEqual(len(result), 1) - self.assertTrue(result[0]["person"]["id"] == person1.uuid, person1.uuid) + self.assertEqual(len(result), 1, result) + self.assertEqual(result[0][0]["id"], person1.uuid, person1.uuid) - @skip("TODO: Bring back when working on actors_in_period") def test_retention_people_first_time(self): _, _, p3, _ = self._create_first_time_retention_events() # even if set to hour 6 it should default to beginning of day and include all pageviews above - target_entity = json.dumps({"id": "$user_signed_up", "type": TREND_FILTER_TYPE_EVENTS}) - result, _ = self.actors_in_period( - OldRetentionFilter( - data={ - "date_to": _date(10, hour=6), - RETENTION_TYPE: RETENTION_FIRST_TIME, - "target_entity": target_entity, + result = self.run_actors_query( + interval=0, + query={ + "dateRange": {"date_to": _date(10, hour=6)}, + "retentionFilter": { + "target_entity": {"id": "$user_signed_up", "type": TREND_FILTER_TYPE_EVENTS}, "returning_entity": {"id": "$pageview", "type": "events"}, - "selected_interval": 0, + "retention_type": RETENTION_FIRST_TIME, }, - team=self.team, - ), - self.team, + }, ) self.assertEqual(len(result), 1) - self.assertIn(result[0]["person"]["id"], [p3.uuid, p3.pk]) - - result, _ = self.actors_in_period( - OldRetentionFilter( - data={ - "date_to": _date(14, hour=6), - RETENTION_TYPE: RETENTION_FIRST_TIME, - "target_entity": target_entity, + self.assertEqual(result[0][0]["id"], p3.uuid) + + result = self.run_actors_query( + interval=0, + query={ + "dateRange": {"date_to": _date(14, hour=6)}, + "retentionFilter": { + "target_entity": {"id": "$user_signed_up", "type": TREND_FILTER_TYPE_EVENTS}, "returning_entity": {"id": "$pageview", "type": "events"}, - "selected_interval": 0, + "retention_type": RETENTION_FIRST_TIME, }, - team=self.team, - ), - self.team, + }, ) self.assertEqual(len(result), 0) @@ -816,7 +819,6 @@ def test_retention_invalid_properties(self): self.validation_error_response("Properties are unparsable!", "invalid_input"), ) - @skip("TODO: Bring back when working on actors_in_period") def test_retention_people_in_period(self): person1 = _create_person(team_id=self.team.pk, distinct_ids=["person1", "alias1"]) person2 = _create_person(team_id=self.team.pk, distinct_ids=["person2"]) @@ -839,43 +841,37 @@ def test_retention_people_in_period(self): ) # even if set to hour 6 it should default to beginning of day and include all pageviews above - result, _ = self.actors_in_period( - OldRetentionFilter( - data={"date_to": _date(10, hour=6), "selected_interval": 2}, - team=self.team, - ), - self.team, + result = self.run_actors_query( + interval=2, + query={ + "dateRange": {"date_to": _date(10, hour=6)}, + }, ) # should be descending order on number of appearances - self.assertIn(result[0]["person"]["id"], [person2.pk, person2.uuid]) - self.assertEqual(result[0]["appearances"], [1, 1, 0, 0, 1, 1, 0, 0, 0]) + self.assertEqual(result[0][0]["id"], person2.uuid) + self.assertCountEqual(result[0][1], [0, 1, 4, 5]) - self.assertIn(result[1]["person"]["id"], [person1.pk, person1.uuid]) - self.assertEqual(result[1]["appearances"], [1, 0, 0, 1, 1, 0, 0, 0, 0]) + self.assertEqual(result[1][0]["id"], person1.uuid) + self.assertCountEqual(result[1][1], [0, 3, 4]) - @skip("TODO: Bring back when working on actors_in_period") - def test_retention_people_in_perieod_first_time(self): + def test_retention_people_in_period_first_time(self): p1, p2, p3, p4 = self._create_first_time_retention_events() # even if set to hour 6 it should default to beginning of day and include all pageviews above - target_entity = json.dumps({"id": "$user_signed_up", "type": TREND_FILTER_TYPE_EVENTS}) - result1, _ = self.actors_in_period( - OldRetentionFilter( - data={ - "date_to": _date(10, hour=6), - RETENTION_TYPE: RETENTION_FIRST_TIME, - "target_entity": target_entity, + result = self.run_actors_query( + interval=0, + query={ + "dateRange": {"date_to": _date(10, hour=6)}, + "retentionFilter": { + "target_entity": {"id": "$user_signed_up", "type": TREND_FILTER_TYPE_EVENTS}, "returning_entity": {"id": "$pageview", "type": "events"}, - "selected_interval": 0, + "retention_type": RETENTION_FIRST_TIME, }, - team=self.team, - ), - self.team, + }, ) - - self.assertEqual(len(result1), 1) - self.assertTrue(result1[0]["person"]["id"] == p3.pk or result1[0]["person"]["id"] == p3.uuid) - self.assertEqual(result1[0]["appearances"], [1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0]) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][0]["id"], p3.uuid) + self.assertCountEqual(result[0][1], [0, 1, 3, 4, 5]) def test_retention_multiple_events(self): _create_person(team_id=self.team.pk, distinct_ids=["person1", "alias1"]) diff --git a/posthog/hogql_queries/insights/trends/query_builder.py b/posthog/hogql_queries/insights/trends/query_builder.py index de75a147b926f..26c287600e455 100644 --- a/posthog/hogql_queries/insights/trends/query_builder.py +++ b/posthog/hogql_queries/insights/trends/query_builder.py @@ -55,6 +55,7 @@ def build_persons_query(self) -> ast.SelectQuery: event_query = self._get_events_subquery(True) event_query.select = [ast.Alias(alias="person_id", expr=ast.Field(chain=["e", "person", "id"]))] + event_query.distinct = True event_query.group_by = None return event_query diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr index 10d5d36c02b51..8088ac81f9d0a 100644 --- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr +++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr @@ -54,7 +54,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -72,17 +72,17 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$bool_prop'), ''), 'null'), '^"|"$', '') AS `properties___$bool_prop` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.person_id, e__pdi__person.id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), ifNull(equals(e__pdi__person.`properties___$bool_prop`, 'x'), 0), and(equals(e.event, 'sign up'), ifNull(in(e__pdi.person_id, - (SELECT cohortpeople.person_id + (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 4)) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version @@ -152,7 +152,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -169,7 +169,7 @@ WHERE equals(person_overrides.team_id, 2) GROUP BY person_overrides.old_person_id) AS e__override ON equals(e.person_id, e__override.old_person_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$bool_prop'), ''), 'null'), '^"|"$', ''), 'x'), 0), and(equals(e.event, 'sign up'), ifNull(in(ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id), - (SELECT cohortpeople.person_id + (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 5)) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version @@ -214,12 +214,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -317,12 +317,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -381,12 +381,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -437,12 +437,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -490,12 +490,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -507,14 +507,14 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT count(DISTINCT actor_id) AS total, - breakdown_value + breakdown_value AS breakdown_value FROM - (SELECT d.timestamp, - e.actor_id, - e.breakdown_value + (SELECT d.timestamp AS timestamp, + e.actor_id AS actor_id, + e.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC')))) AS numbers) AS d @@ -575,12 +575,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -592,14 +592,14 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT count(DISTINCT actor_id) AS total, - breakdown_value + breakdown_value AS breakdown_value FROM - (SELECT d.timestamp, - e.actor_id, - e.breakdown_value + (SELECT d.timestamp AS timestamp, + e.actor_id AS actor_id, + e.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC')))) AS numbers) AS d @@ -675,17 +675,17 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'name'), ''), 'null'), '^"|"$', '') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.person_id, e__pdi__person.id) WHERE and(equals(e.team_id, 2), equals(e.event, '$pageview'), and(or(ifNull(equals(e__pdi__person.properties___name, 'p1'), 0), ifNull(equals(e__pdi__person.properties___name, 'p2'), 0), ifNull(equals(e__pdi__person.properties___name, 'p3'), 0)), ifNull(in(e__pdi.person_id, - (SELECT cohortpeople.person_id + (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 24)) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version @@ -705,12 +705,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -723,11 +723,11 @@ ORDER BY sec.breakdown_value ASC, day_start ASC UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start, - breakdown_value + breakdown_value AS breakdown_value FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts, - e.breakdown_value + e.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC')))) AS numbers) AS d @@ -744,17 +744,17 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'name'), ''), 'null'), '^"|"$', '') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.person_id, e__pdi__person.id) WHERE and(equals(e.team_id, 2), and(and(equals(e.event, '$pageview'), and(or(ifNull(equals(e__pdi__person.properties___name, 'p1'), 0), ifNull(equals(e__pdi__person.properties___name, 'p2'), 0), ifNull(equals(e__pdi__person.properties___name, 'p3'), 0)), ifNull(in(e__pdi.person_id, - (SELECT cohortpeople.person_id + (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 24)) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version @@ -807,12 +807,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -893,12 +893,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -963,12 +963,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -1032,12 +1032,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -1124,7 +1124,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-02 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1142,11 +1142,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'name'), ''), 'null'), '^"|"$', '') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -1207,7 +1207,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-02 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1233,7 +1233,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1314,7 +1314,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1357,12 +1357,12 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop`, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'filter_prop'), ''), 'null'), '^"|"$', '') AS properties___filter_prop FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -1383,12 +1383,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -1401,11 +1401,11 @@ ORDER BY sec.breakdown_value ASC, day_start ASC UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start, - breakdown_value + breakdown_value AS breakdown_value FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts, - e.breakdown_value + e.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC'))), toIntervalDay(30)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))) AS numbers) AS d @@ -1422,12 +1422,12 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop`, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'filter_prop'), ''), 'null'), '^"|"$', '') AS properties___filter_prop FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -1473,12 +1473,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -1491,11 +1491,11 @@ ORDER BY sec.breakdown_value ASC, day_start ASC UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start, - breakdown_value + breakdown_value AS breakdown_value FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts, - e.breakdown_value + e.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC'))), toIntervalDay(30)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))) AS numbers) AS d @@ -1534,7 +1534,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2022-11-30 23:59:59', 6, 'US/Pacific')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -1589,7 +1589,7 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), and(equals(e.event, 'sign up'), ifNull(in(e__pdi.person_id, - (SELECT cohortpeople.person_id + (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 37)) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version @@ -1609,12 +1609,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -1637,7 +1637,7 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), and(equals(e.event, 'sign up'), ifNull(in(e__pdi.person_id, - (SELECT cohortpeople.person_id + (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 37)) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version @@ -1688,7 +1688,7 @@ WHERE equals(person_overrides.team_id, 2) GROUP BY person_overrides.old_person_id) AS e__override ON equals(e.person_id, e__override.old_person_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), and(equals(e.event, 'sign up'), ifNull(in(ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id), - (SELECT cohortpeople.person_id + (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 38)) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version @@ -1708,12 +1708,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -1735,7 +1735,7 @@ WHERE equals(person_overrides.team_id, 2) GROUP BY person_overrides.old_person_id) AS e__override ON equals(e.person_id, e__override.old_person_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:01:01', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), and(equals(e.event, 'sign up'), ifNull(in(ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id), - (SELECT cohortpeople.person_id + (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 38)) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version @@ -1758,7 +1758,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1776,11 +1776,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'name'), ''), 'null'), '^"|"$', '') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -1801,7 +1801,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1819,11 +1819,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'name'), ''), 'null'), '^"|"$', '') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -1844,7 +1844,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1870,7 +1870,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1888,11 +1888,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, nullIf(nullIf(person.pmat_name, ''), 'null') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -1913,7 +1913,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1939,7 +1939,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -1957,11 +1957,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, nullIf(nullIf(person.pmat_name, ''), 'null') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -1996,7 +1996,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-03 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -2022,7 +2022,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-03 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -2054,7 +2054,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -2080,7 +2080,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -2119,7 +2119,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -2129,7 +2129,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp @@ -2171,7 +2171,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -2214,12 +2214,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 13:01:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -2266,7 +2266,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), toIntervalDay(numbers.number)) AS day_start @@ -2292,7 +2292,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), toIntervalDay(numbers.number)) AS day_start @@ -2331,7 +2331,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), toIntervalDay(numbers.number)) AS day_start @@ -2341,7 +2341,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), toIntervalDay(numbers.number)) AS timestamp @@ -2383,7 +2383,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), toIntervalDay(numbers.number)) AS day_start @@ -2426,12 +2426,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 06:01:01', 6, 'America/Phoenix')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), 0)) AS numbers @@ -2478,7 +2478,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), toIntervalDay(numbers.number)) AS day_start @@ -2504,7 +2504,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), toIntervalDay(numbers.number)) AS day_start @@ -2543,7 +2543,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), toIntervalDay(numbers.number)) AS day_start @@ -2553,7 +2553,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), toIntervalDay(numbers.number)) AS timestamp @@ -2595,7 +2595,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), toIntervalDay(numbers.number)) AS day_start @@ -2638,12 +2638,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 22:01:01', 6, 'Asia/Tokyo')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), 0)) AS numbers @@ -2690,7 +2690,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfHour(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 10:59:59', 6, 'UTC'))), toIntervalHour(numbers.number)) AS day_start @@ -2729,7 +2729,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfHour(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 10:59:59', 6, 'UTC'))), toIntervalHour(numbers.number)) AS day_start @@ -2755,7 +2755,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfHour(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 10:59:59', 6, 'America/Phoenix'))), toIntervalHour(numbers.number)) AS day_start @@ -2794,7 +2794,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfHour(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 10:59:59', 6, 'America/Phoenix'))), toIntervalHour(numbers.number)) AS day_start @@ -2820,7 +2820,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfHour(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 10:59:59', 6, 'Asia/Tokyo'))), toIntervalHour(numbers.number)) AS day_start @@ -2859,7 +2859,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfHour(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 10:59:59', 6, 'Asia/Tokyo'))), toIntervalHour(numbers.number)) AS day_start @@ -2885,7 +2885,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-26 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -2911,7 +2911,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-26 23:59:59', 6, 'UTC')), 3), toIntervalWeek(numbers.number)) AS day_start @@ -2937,7 +2937,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-26 23:59:59', 6, 'America/Phoenix')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -2963,7 +2963,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-26 23:59:59', 6, 'America/Phoenix')), 3), toIntervalWeek(numbers.number)) AS day_start @@ -2989,7 +2989,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-26 23:59:59', 6, 'Asia/Tokyo')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -3015,7 +3015,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-26 23:59:59', 6, 'Asia/Tokyo')), 3), toIntervalWeek(numbers.number)) AS day_start @@ -3050,13 +3050,13 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$os'), ''), 'null'), '^"|"$', '') AS `properties___$os`, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -3077,12 +3077,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -3105,13 +3105,13 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$os'), ''), 'null'), '^"|"$', '') AS `properties___$os`, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -3144,13 +3144,13 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$os'), ''), 'null'), '^"|"$', '') AS `properties___$os`, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -3171,12 +3171,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -3199,13 +3199,13 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$os'), ''), 'null'), '^"|"$', '') AS `properties___$os`, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -3229,7 +3229,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3255,7 +3255,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3273,11 +3273,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -3307,11 +3307,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -3332,12 +3332,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 13:00:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -3360,11 +3360,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -3388,7 +3388,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3398,7 +3398,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp @@ -3427,7 +3427,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3437,7 +3437,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp @@ -3483,12 +3483,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 13:00:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -3522,7 +3522,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3548,7 +3548,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3590,18 +3590,18 @@ groupArray(count) AS total, ifNull(toString(breakdown_value), '') AS breakdown_value FROM - (SELECT day_start, + (SELECT day_start AS day_start, sum(count) OVER (PARTITION BY breakdown_value ORDER BY day_start ASC) AS count, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:00:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -3664,18 +3664,18 @@ groupArray(count) AS total, ifNull(toString(breakdown_value), '') AS breakdown_value FROM - (SELECT day_start, + (SELECT day_start AS day_start, sum(count) OVER (PARTITION BY breakdown_value ORDER BY day_start ASC) AS count, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:00:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -3729,12 +3729,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -3746,10 +3746,10 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT quantile(0.5)(session_duration) AS total, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT any(e__session.duration) AS session_duration, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS breakdown_value @@ -3793,12 +3793,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3810,10 +3810,10 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT quantile(0.5)(session_duration) AS total, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT any(e__session.duration) AS session_duration, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS breakdown_value @@ -3843,7 +3843,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3869,7 +3869,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3895,7 +3895,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3919,12 +3919,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3936,7 +3936,7 @@ FROM (SELECT sum(total) AS count FROM - (SELECT total + (SELECT total AS total FROM (SELECT avg(total) AS total FROM @@ -3967,12 +3967,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -3984,7 +3984,7 @@ FROM (SELECT sum(total) AS count FROM - (SELECT total + (SELECT total AS total FROM (SELECT avg(total) AS total FROM @@ -4024,12 +4024,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4041,13 +4041,13 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - breakdown_value + breakdown_value AS breakdown_value FROM - (SELECT total, - breakdown_value + (SELECT total AS total, + breakdown_value AS breakdown_value FROM (SELECT avg(total) AS total, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT count(e.uuid) AS total, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '') AS breakdown_value @@ -4084,18 +4084,18 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), 0)) AS numbers UNION ALL SELECT 0 AS total, toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))) AS day_start - UNION ALL SELECT total, - day_start + UNION ALL SELECT total AS total, + day_start AS day_start FROM (SELECT avg(total) AS total, - day_start + day_start AS day_start FROM (SELECT count(e.uuid) AS total, toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start @@ -4131,18 +4131,18 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))), 0)) AS numbers UNION ALL SELECT 0 AS total, toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))) AS day_start - UNION ALL SELECT total, - day_start + UNION ALL SELECT total AS total, + day_start AS day_start FROM (SELECT avg(total) AS total, - day_start + day_start AS day_start FROM (SELECT count(e.uuid) AS total, toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start @@ -4170,12 +4170,12 @@ SELECT groupArray(day_start) AS date, groupArray(count) AS total FROM - (SELECT day_start, + (SELECT day_start AS day_start, sum(count) OVER ( ORDER BY day_start ASC) AS count FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4210,11 +4210,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -4232,12 +4232,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -4249,10 +4249,10 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT quantile(0.5)(session_duration) AS total, - breakdown_value + breakdown_value AS breakdown_value FROM (SELECT any(e__session.duration) AS session_duration, e__pdi__person.`properties___$some_prop` AS breakdown_value @@ -4271,11 +4271,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -4299,7 +4299,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -4323,12 +4323,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -4362,12 +4362,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4403,7 +4403,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -4411,7 +4411,7 @@ UNION ALL SELECT 0 AS total, toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:00:01', 6, 'UTC')), 0) AS day_start UNION ALL SELECT quantile(0.5)(session_duration) AS total, - day_start + day_start AS day_start FROM (SELECT any(e__session.duration) AS session_duration, toStartOfWeek(toTimeZone(e.timestamp, 'UTC'), 0) AS day_start @@ -4441,7 +4441,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4449,7 +4449,7 @@ UNION ALL SELECT 0 AS total, toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:00:01', 6, 'UTC'))) AS day_start UNION ALL SELECT quantile(0.5)(session_duration) AS total, - day_start + day_start AS day_start FROM (SELECT any(e__session.duration) AS session_duration, toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start @@ -4496,12 +4496,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('week', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:00:01', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -4513,8 +4513,8 @@ JOIN breakdown_value AS breakdown_value) AS sec ORDER BY sec.breakdown_value ASC, day_start ASC UNION ALL SELECT quantile(0.5)(session_duration) AS total, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT any(e__session.duration) AS session_duration, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS breakdown_value, @@ -4566,12 +4566,12 @@ ifNull(toString(breakdown_value), '') AS breakdown_value FROM (SELECT sum(total) AS count, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT 0 AS total, ticks.day_start AS day_start, - sec.breakdown_value + sec.breakdown_value AS breakdown_value FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 13:00:05', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) AS numbers @@ -4583,8 +4583,8 @@ JOIN breakdown_value AS breakdown_value) AS sec ORDER BY sec.breakdown_value ASC, day_start ASC UNION ALL SELECT quantile(0.5)(session_duration) AS total, - day_start, - breakdown_value + day_start AS day_start, + breakdown_value AS breakdown_value FROM (SELECT any(e__session.duration) AS session_duration, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS breakdown_value, @@ -4617,12 +4617,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4636,8 +4636,8 @@ FROM (SELECT count(DISTINCT actor_id) AS total FROM - (SELECT d.timestamp, - e.actor_id + (SELECT d.timestamp AS timestamp, + e.actor_id AS actor_id FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC')))) AS numbers) AS d @@ -4675,12 +4675,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-08 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4694,8 +4694,8 @@ FROM (SELECT count(DISTINCT actor_id) AS total FROM - (SELECT d.timestamp, - e.actor_id + (SELECT d.timestamp AS timestamp, + e.actor_id AS actor_id FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-08 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-08 23:59:59', 6, 'UTC')))) AS numbers) AS d @@ -4733,12 +4733,12 @@ ' SELECT sum(count) AS total, - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 1 AS group_key, groupArray(day_start) AS day_start FROM - (SELECT day_start + (SELECT day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-08 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4752,8 +4752,8 @@ FROM (SELECT count(DISTINCT actor_id) AS total FROM - (SELECT d.timestamp, - e.actor_id + (SELECT d.timestamp AS timestamp, + e.actor_id AS actor_id FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-08 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-08 23:59:59', 6, 'UTC')))) AS numbers) AS d @@ -4793,7 +4793,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4803,7 +4803,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp @@ -4845,7 +4845,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'America/Phoenix'))), toIntervalDay(numbers.number)) AS day_start @@ -4855,7 +4855,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'America/Phoenix'))), toIntervalDay(numbers.number)) AS timestamp @@ -4897,7 +4897,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'Asia/Tokyo'))), toIntervalDay(numbers.number)) AS day_start @@ -4907,7 +4907,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-19 23:59:59', 6, 'Asia/Tokyo'))), toIntervalDay(numbers.number)) AS timestamp @@ -4949,7 +4949,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -4959,7 +4959,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp @@ -4976,11 +4976,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'name'), ''), 'null'), '^"|"$', '') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -5005,7 +5005,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start @@ -5015,7 +5015,7 @@ UNION ALL SELECT counts AS total, toStartOfDay(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp @@ -5032,11 +5032,11 @@ GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) INNER JOIN - (SELECT person.id, + (SELECT person.id AS id, nullIf(nullIf(person.pmat_name, ''), 'null') AS properties___name FROM person WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id, max(person.version) AS version + (SELECT person.id AS id, max(person.version) AS version FROM person WHERE equals(person.team_id, 2) GROUP BY person.id @@ -5061,7 +5061,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfHour(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-09 17:00:00', 6, 'UTC'))), toIntervalHour(numbers.number)) AS day_start @@ -5071,7 +5071,7 @@ UNION ALL SELECT counts AS total, toStartOfHour(timestamp) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfHour(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-09 17:00:00', 6, 'UTC'))), toIntervalHour(numbers.number)) AS timestamp @@ -5113,7 +5113,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-18 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -5123,7 +5123,7 @@ UNION ALL SELECT counts AS total, toStartOfWeek(timestamp, 0) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-18 23:59:59', 6, 'UTC')), 0), toIntervalWeek(numbers.number)) AS timestamp @@ -5165,7 +5165,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-18 23:59:59', 6, 'America/Phoenix')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -5175,7 +5175,7 @@ UNION ALL SELECT counts AS total, toStartOfWeek(timestamp, 0) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-18 23:59:59', 6, 'America/Phoenix')), 0), toIntervalWeek(numbers.number)) AS timestamp @@ -5217,7 +5217,7 @@ groupArray(count) AS total FROM (SELECT sum(total) AS count, - day_start + day_start AS day_start FROM (SELECT 0 AS total, minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-18 23:59:59', 6, 'Asia/Tokyo')), 0), toIntervalWeek(numbers.number)) AS day_start @@ -5227,7 +5227,7 @@ UNION ALL SELECT counts AS total, toStartOfWeek(timestamp, 0) AS day_start FROM - (SELECT d.timestamp, + (SELECT d.timestamp AS timestamp, count(DISTINCT e.actor_id) AS counts FROM (SELECT minus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-18 23:59:59', 6, 'Asia/Tokyo')), 0), toIntervalWeek(numbers.number)) AS timestamp diff --git a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py index 8238990bbfa50..1f8f39ee41f06 100644 --- a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py +++ b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py @@ -275,6 +275,7 @@ def _breakdown_filter(_filter: Dict): "breakdown": _filter.get("breakdown"), "breakdown_normalize_url": _filter.get("breakdown_normalize_url"), "breakdown_group_type_index": _filter.get("breakdown_group_type_index"), + "breakdown_hide_other_aggregation": _filter.get("breakdown_hide_other_aggregation"), "breakdown_histogram_bin_count": _filter.get("breakdown_histogram_bin_count") if _insight_type(_filter) == "TRENDS" else None, diff --git a/posthog/hogql_queries/persons_query_runner.py b/posthog/hogql_queries/persons_query_runner.py index a5a638c21f903..2b97a21811f65 100644 --- a/posthog/hogql_queries/persons_query_runner.py +++ b/posthog/hogql_queries/persons_query_runner.py @@ -1,23 +1,53 @@ from datetime import timedelta -from typing import List, cast, Literal, Dict, Any -from django.db.models.query import Prefetch - +from typing import List, Generator, Sequence, Iterator, Optional from posthog.hogql import ast -from posthog.hogql.constants import get_max_limit_for_context, get_default_limit_for_context from posthog.hogql.parser import parse_expr, parse_order_expr -from posthog.hogql.property import property_to_expr, has_aggregation -from posthog.hogql.query import execute_hogql_query +from posthog.hogql.property import has_aggregation +from posthog.hogql_queries.actor_strategies import ActorStrategy, PersonStrategy, GroupStrategy +from posthog.hogql_queries.insights.insight_persons_query_runner import InsightPersonsQueryRunner +from posthog.hogql_queries.insights.paginators import HogQLHasMorePaginator from posthog.hogql_queries.query_runner import QueryRunner, get_query_runner from posthog.schema import PersonsQuery, PersonsQueryResponse -from posthog.models.person import Person class PersonsQueryRunner(QueryRunner): query: PersonsQuery query_type = PersonsQuery + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.paginator = HogQLHasMorePaginator.from_limit_context( + limit_context=self.limit_context, limit=self.query.limit, offset=self.query.offset + ) + self.source_query_runner: Optional[QueryRunner] = None + + if self.query.source: + self.source_query_runner = get_query_runner(self.query.source, self.team, self.timings, self.limit_context) + + self.strategy = self.determine_strategy() + + @property + def group_type_index(self) -> int | None: + if not self.source_query_runner or not isinstance(self.source_query_runner, InsightPersonsQueryRunner): + return None + + return self.source_query_runner.group_type_index + + def determine_strategy(self) -> ActorStrategy: + if self.group_type_index is not None: + return GroupStrategy(self.group_type_index, team=self.team, query=self.query, paginator=self.paginator) + return PersonStrategy(team=self.team, query=self.query, paginator=self.paginator) + + def enrich_with_actors(self, results, actor_column_index, actors_lookup) -> Generator[List, None, None]: + for result in results: + new_row = list(result) + actor_id = str(result[actor_column_index]) + actor = actors_lookup.get(actor_id) + new_row[actor_column_index] = actor if actor else {"id": actor_id} + yield new_row + def calculate(self) -> PersonsQueryResponse: - response = execute_hogql_query( + response = self.paginator.execute_hogql_query( query_type="PersonsQuery", query=self.to_query(), team=self.team, @@ -25,99 +55,60 @@ def calculate(self) -> PersonsQueryResponse: modifiers=self.modifiers, ) input_columns = self.input_columns() - if "person" in input_columns: - person_column_index = input_columns.index("person") - person_ids = [str(result[person_column_index]) for result in response.results] - pg_persons = { - str(p.uuid): p - for p in Person.objects.filter(team_id=self.team.pk, persondistinctid__team_id=self.team.pk) - .filter(uuid__in=person_ids) - .prefetch_related(Prefetch("persondistinctid_set", to_attr="distinct_ids_cache")) - } - - for index, result in enumerate(response.results): - response.results[index] = list(result) - person_id = str(result[person_column_index]) - new_result: Dict[str, Any] = {"id": person_id} - person = pg_persons.get(person_id) - if person: - new_result["distinct_ids"] = person.distinct_ids - new_result["properties"] = person.properties - new_result["created_at"] = person.created_at - new_result["is_identified"] = person.is_identified - response.results[index][person_column_index] = new_result - - has_more = len(response.results) > self.query_limit() + missing_actors_count = None + results: Sequence[List] | Iterator[List] = self.paginator.results + + enrich_columns = filter(lambda column: column in ("person", "group"), input_columns) + for column_name in enrich_columns: + actor_ids = (row[input_columns.index(column_name)] for row in self.paginator.results) + actors_lookup = self.strategy.get_actors(actor_ids) + missing_actors_count = len(self.paginator.results) - len(actors_lookup) + results = self.enrich_with_actors(results, input_columns.index(column_name), actors_lookup) + return PersonsQueryResponse( - # we added +1 before for pagination, remove the last element if there's more - results=response.results[:-1] if has_more else response.results, + results=results, timings=response.timings, - types=[type for _, type in response.types], - columns=self.input_columns(), + types=[t for _, t in response.types] if response.types else None, + columns=input_columns, hogql=response.hogql, - hasMore=has_more, + missing_actors_count=missing_actors_count, + **self.paginator.response_params(), ) - def filter_conditions(self) -> List[ast.Expr]: - where_exprs: List[ast.Expr] = [] - - if self.query.source: - source = self.query.source - try: - source_query_runner = get_query_runner(source, self.team, self.timings) - source_query = source_query_runner.to_persons_query() - where_exprs.append( - ast.CompareOperation( - left=ast.Field(chain=["id"]), - op=ast.CompareOperationOp.In, - right=source_query, - ) - ) - except NotImplementedError: - raise ValueError(f"Queries of type '{source.kind}' are not implemented as a PersonsQuery sources.") - - if self.query.properties: - where_exprs.append(property_to_expr(self.query.properties, self.team, scope="person")) - - if self.query.fixedProperties: - where_exprs.append(property_to_expr(self.query.fixedProperties, self.team, scope="person")) - - if self.query.search is not None and self.query.search != "": - where_exprs.append( - ast.Or( - exprs=[ - ast.CompareOperation( - op=ast.CompareOperationOp.ILike, - left=ast.Field(chain=["properties", "email"]), - right=ast.Constant(value=f"%{self.query.search}%"), - ), - ast.CompareOperation( - op=ast.CompareOperationOp.ILike, - left=ast.Field(chain=["properties", "name"]), - right=ast.Constant(value=f"%{self.query.search}%"), - ), - ast.CompareOperation( - op=ast.CompareOperationOp.ILike, - left=ast.Call(name="toString", args=[ast.Field(chain=["id"])]), - right=ast.Constant(value=f"%{self.query.search}%"), - ), - ast.CompareOperation( - op=ast.CompareOperationOp.ILike, - left=ast.Field(chain=["pdi", "distinct_id"]), - right=ast.Constant(value=f"%{self.query.search}%"), - ), - ] - ) - ) - return where_exprs - def input_columns(self) -> List[str]: - return self.query.select or ["person", "id", "created_at", "person.$delete"] - - def query_limit(self) -> int: - max_rows = get_max_limit_for_context(self.limit_context) - default_rows = get_default_limit_for_context(self.limit_context) - return min(max_rows, default_rows if self.query.limit is None else self.query.limit) + if self.query.select: + return self.query.select + + return self.strategy.input_columns() + + def source_id_column(self, source_query: ast.SelectQuery) -> List[str]: + # Figure out the id column of the source query, first column that has id in the name + for column in source_query.select: + if isinstance(column, ast.Field) and any("id" in part.lower() for part in column.chain): + return column.chain + raise ValueError("Source query must have an id column") + + def source_table_join(self) -> ast.JoinExpr: + assert self.source_query_runner is not None # For type checking + source_query = self.source_query_runner.to_persons_query() + source_id_chain = self.source_id_column(source_query) + source_alias = "source" + + return ast.JoinExpr( + table=ast.Field(chain=[self.strategy.origin]), + next_join=ast.JoinExpr( + table=source_query, + join_type="INNER JOIN", + alias=source_alias, + constraint=ast.JoinConstraint( + expr=ast.CompareOperation( + op=ast.CompareOperationOp.Eq, + left=ast.Field(chain=[self.strategy.origin, self.strategy.origin_id]), + right=ast.Field(chain=[source_alias, *source_id_chain]), + ) + ), + ), + ) def to_query(self) -> ast.SelectQuery: with self.timings.measure("columns"): @@ -127,8 +118,8 @@ def to_query(self) -> ast.SelectQuery: for expr in self.input_columns(): if expr == "person.$delete": column = ast.Constant(value=1) - elif expr == "person": - column = ast.Field(chain=["id"]) + elif expr == self.strategy.field: + column = ast.Field(chain=[self.strategy.origin_id]) else: column = parse_expr(expr) columns.append(column) @@ -139,7 +130,7 @@ def to_query(self) -> ast.SelectQuery: has_any_aggregation = len(aggregations) > 0 with self.timings.measure("filters"): - filter_conditions = self.filter_conditions() + filter_conditions = self.strategy.filter_conditions() where_list = [expr for expr in filter_conditions if not has_aggregation(expr)] if len(where_list) == 0: where = None @@ -158,21 +149,9 @@ def to_query(self) -> ast.SelectQuery: with self.timings.measure("order"): if self.query.orderBy is not None: - if self.query.orderBy in [["person"], ["person DESC"], ["person ASC"]]: - order_property = ( - "email" - if self.team.person_display_name_properties is None - else self.team.person_display_name_properties[0] - ) - order_by = [ - ast.OrderExpr( - expr=ast.Field(chain=["properties", order_property]), - order=cast( - Literal["ASC", "DESC"], - "DESC" if self.query.orderBy[0] == "person DESC" else "ASC", - ), - ) - ] + strategy_order_by = self.strategy.order_by() + if strategy_order_by is not None: + order_by = strategy_order_by else: order_by = [parse_order_expr(column, timings=self.timings) for column in self.query.orderBy] elif "count()" in self.input_columns(): @@ -186,21 +165,19 @@ def to_query(self) -> ast.SelectQuery: else: order_by = [] - with self.timings.measure("limit"): - # adding +1 to the limit to check if there's a "next page" after the requested results - limit = self.query_limit() + 1 - offset = 0 if self.query.offset is None else self.query.offset - with self.timings.measure("select"): + if self.query.source: + join_expr = self.source_table_join() + else: + join_expr = ast.JoinExpr(table=ast.Field(chain=[self.strategy.origin])) + stmt = ast.SelectQuery( select=columns, - select_from=ast.JoinExpr(table=ast.Field(chain=["persons"])), + select_from=join_expr, where=where, having=having, group_by=group_by if has_any_aggregation else None, order_by=order_by, - limit=ast.Constant(value=limit), - offset=ast.Constant(value=offset), ) return stmt diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py index 85c2d29372676..afe2b0aa84544 100644 --- a/posthog/hogql_queries/query_runner.py +++ b/posthog/hogql_queries/query_runner.py @@ -59,6 +59,8 @@ class QueryResponse(BaseModel, Generic[DataT]): columns: Optional[List[str]] = None hogql: Optional[str] = None hasMore: Optional[bool] = None + limit: Optional[int] = None + offset: Optional[int] = None class CachedQueryResponse(QueryResponse): diff --git a/posthog/hogql_queries/test/__snapshots__/test_sessions_timeline_query_runner.ambr b/posthog/hogql_queries/test/__snapshots__/test_sessions_timeline_query_runner.ambr index b73f7380c6b6b..6f369e808e993 100644 --- a/posthog/hogql_queries/test/__snapshots__/test_sessions_timeline_query_runner.ambr +++ b/posthog/hogql_queries/test/__snapshots__/test_sessions_timeline_query_runner.ambr @@ -1,34 +1,36 @@ # name: TestSessionsTimelineQueryRunner.test_before_and_after ' - SELECT e.uuid, - e.timestamp, - e.event, - e.properties, - e.distinct_id, - e.elements_chain, + SELECT e.uuid AS uuid, + e.timestamp AS timestamp, + e.event AS event, + e.properties AS properties, + e.distinct_id AS distinct_id, + e.elements_chain AS elements_chain, e.session_id AS formal_session_id, first_value(e.uuid) OVER (PARTITION BY tuple(e.person_id, e.session_id_flip_index) ORDER BY toInt64(e.timestamp) ASC RANGE BETWEEN 1800 PRECEDING AND CURRENT ROW) AS informal_session_uuid, dateDiff('s', sre.start_time, sre.end_time) AS recording_duration_s FROM - (SELECT uuid, - person_id, timestamp, event, - properties, - distinct_id, - elements_chain, - session_id, - prev_session_id, - sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) - and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id - ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index + (SELECT uuid AS uuid, + person_id AS person_id, + timestamp AS timestamp, + event AS event, + properties AS properties, + distinct_id AS distinct_id, + elements_chain AS elements_chain, + session_id AS session_id, + prev_session_id AS prev_session_id, + sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) + and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id + ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index FROM - (SELECT events.uuid, + (SELECT events.uuid AS uuid, events__pdi.person_id AS person_id, toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.event, - events.properties, - events.distinct_id, - events.elements_chain, + events.event AS event, + events.properties AS properties, + events.distinct_id AS distinct_id, + events.elements_chain AS elements_chain, events.`$session_id` AS session_id, lagInFrame(events.`$session_id`, 1) OVER (PARTITION BY person_id ORDER BY timestamp ASC) AS prev_session_id @@ -46,7 +48,7 @@ LEFT JOIN (SELECT toTimeZone(session_replay_events.start_time, 'UTC') AS start_time, toTimeZone(session_replay_events.end_time, 'UTC') AS end_time, - session_replay_events.session_id + session_replay_events.session_id AS session_id FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, max(session_replay_events.max_last_timestamp) AS end_time, @@ -62,35 +64,37 @@ --- # name: TestSessionsTimelineQueryRunner.test_before_and_after_defaults ' - SELECT e.uuid, - e.timestamp, - e.event, - e.properties, - e.distinct_id, - e.elements_chain, + SELECT e.uuid AS uuid, + e.timestamp AS timestamp, + e.event AS event, + e.properties AS properties, + e.distinct_id AS distinct_id, + e.elements_chain AS elements_chain, e.session_id AS formal_session_id, first_value(e.uuid) OVER (PARTITION BY tuple(e.person_id, e.session_id_flip_index) ORDER BY toInt64(e.timestamp) ASC RANGE BETWEEN 1800 PRECEDING AND CURRENT ROW) AS informal_session_uuid, dateDiff('s', sre.start_time, sre.end_time) AS recording_duration_s FROM - (SELECT uuid, - person_id, timestamp, event, - properties, - distinct_id, - elements_chain, - session_id, - prev_session_id, - sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) - and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id - ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index + (SELECT uuid AS uuid, + person_id AS person_id, + timestamp AS timestamp, + event AS event, + properties AS properties, + distinct_id AS distinct_id, + elements_chain AS elements_chain, + session_id AS session_id, + prev_session_id AS prev_session_id, + sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) + and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id + ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index FROM - (SELECT events.uuid, + (SELECT events.uuid AS uuid, events__pdi.person_id AS person_id, toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.event, - events.properties, - events.distinct_id, - events.elements_chain, + events.event AS event, + events.properties AS properties, + events.distinct_id AS distinct_id, + events.elements_chain AS elements_chain, events.`$session_id` AS session_id, lagInFrame(events.`$session_id`, 1) OVER (PARTITION BY person_id ORDER BY timestamp ASC) AS prev_session_id @@ -108,7 +112,7 @@ LEFT JOIN (SELECT toTimeZone(session_replay_events.start_time, 'UTC') AS start_time, toTimeZone(session_replay_events.end_time, 'UTC') AS end_time, - session_replay_events.session_id + session_replay_events.session_id AS session_id FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, max(session_replay_events.max_last_timestamp) AS end_time, @@ -124,35 +128,37 @@ --- # name: TestSessionsTimelineQueryRunner.test_event_limit_and_has_more ' - SELECT e.uuid, - e.timestamp, - e.event, - e.properties, - e.distinct_id, - e.elements_chain, + SELECT e.uuid AS uuid, + e.timestamp AS timestamp, + e.event AS event, + e.properties AS properties, + e.distinct_id AS distinct_id, + e.elements_chain AS elements_chain, e.session_id AS formal_session_id, first_value(e.uuid) OVER (PARTITION BY tuple(e.person_id, e.session_id_flip_index) ORDER BY toInt64(e.timestamp) ASC RANGE BETWEEN 1800 PRECEDING AND CURRENT ROW) AS informal_session_uuid, dateDiff('s', sre.start_time, sre.end_time) AS recording_duration_s FROM - (SELECT uuid, - person_id, timestamp, event, - properties, - distinct_id, - elements_chain, - session_id, - prev_session_id, - sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) - and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id - ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index + (SELECT uuid AS uuid, + person_id AS person_id, + timestamp AS timestamp, + event AS event, + properties AS properties, + distinct_id AS distinct_id, + elements_chain AS elements_chain, + session_id AS session_id, + prev_session_id AS prev_session_id, + sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) + and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id + ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index FROM - (SELECT events.uuid, + (SELECT events.uuid AS uuid, events__pdi.person_id AS person_id, toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.event, - events.properties, - events.distinct_id, - events.elements_chain, + events.event AS event, + events.properties AS properties, + events.distinct_id AS distinct_id, + events.elements_chain AS elements_chain, events.`$session_id` AS session_id, lagInFrame(events.`$session_id`, 1) OVER (PARTITION BY person_id ORDER BY timestamp ASC) AS prev_session_id @@ -170,7 +176,7 @@ LEFT JOIN (SELECT toTimeZone(session_replay_events.start_time, 'UTC') AS start_time, toTimeZone(session_replay_events.end_time, 'UTC') AS end_time, - session_replay_events.session_id + session_replay_events.session_id AS session_id FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, max(session_replay_events.max_last_timestamp) AS end_time, @@ -186,35 +192,37 @@ --- # name: TestSessionsTimelineQueryRunner.test_formal_and_informal_sessions_global ' - SELECT e.uuid, - e.timestamp, - e.event, - e.properties, - e.distinct_id, - e.elements_chain, + SELECT e.uuid AS uuid, + e.timestamp AS timestamp, + e.event AS event, + e.properties AS properties, + e.distinct_id AS distinct_id, + e.elements_chain AS elements_chain, e.session_id AS formal_session_id, first_value(e.uuid) OVER (PARTITION BY tuple(e.person_id, e.session_id_flip_index) ORDER BY toInt64(e.timestamp) ASC RANGE BETWEEN 1800 PRECEDING AND CURRENT ROW) AS informal_session_uuid, dateDiff('s', sre.start_time, sre.end_time) AS recording_duration_s FROM - (SELECT uuid, - person_id, timestamp, event, - properties, - distinct_id, - elements_chain, - session_id, - prev_session_id, - sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) - and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id - ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index + (SELECT uuid AS uuid, + person_id AS person_id, + timestamp AS timestamp, + event AS event, + properties AS properties, + distinct_id AS distinct_id, + elements_chain AS elements_chain, + session_id AS session_id, + prev_session_id AS prev_session_id, + sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) + and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id + ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index FROM - (SELECT events.uuid, + (SELECT events.uuid AS uuid, events__pdi.person_id AS person_id, toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.event, - events.properties, - events.distinct_id, - events.elements_chain, + events.event AS event, + events.properties AS properties, + events.distinct_id AS distinct_id, + events.elements_chain AS elements_chain, events.`$session_id` AS session_id, lagInFrame(events.`$session_id`, 1) OVER (PARTITION BY person_id ORDER BY timestamp ASC) AS prev_session_id @@ -232,7 +240,7 @@ LEFT JOIN (SELECT toTimeZone(session_replay_events.start_time, 'UTC') AS start_time, toTimeZone(session_replay_events.end_time, 'UTC') AS end_time, - session_replay_events.session_id + session_replay_events.session_id AS session_id FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, max(session_replay_events.max_last_timestamp) AS end_time, @@ -248,35 +256,37 @@ --- # name: TestSessionsTimelineQueryRunner.test_formal_session_with_recording ' - SELECT e.uuid, - e.timestamp, - e.event, - e.properties, - e.distinct_id, - e.elements_chain, + SELECT e.uuid AS uuid, + e.timestamp AS timestamp, + e.event AS event, + e.properties AS properties, + e.distinct_id AS distinct_id, + e.elements_chain AS elements_chain, e.session_id AS formal_session_id, first_value(e.uuid) OVER (PARTITION BY tuple(e.person_id, e.session_id_flip_index) ORDER BY toInt64(e.timestamp) ASC RANGE BETWEEN 1800 PRECEDING AND CURRENT ROW) AS informal_session_uuid, dateDiff('s', sre.start_time, sre.end_time) AS recording_duration_s FROM - (SELECT uuid, - person_id, timestamp, event, - properties, - distinct_id, - elements_chain, - session_id, - prev_session_id, - sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) - and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id - ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index + (SELECT uuid AS uuid, + person_id AS person_id, + timestamp AS timestamp, + event AS event, + properties AS properties, + distinct_id AS distinct_id, + elements_chain AS elements_chain, + session_id AS session_id, + prev_session_id AS prev_session_id, + sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) + and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id + ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index FROM - (SELECT events.uuid, + (SELECT events.uuid AS uuid, events__pdi.person_id AS person_id, toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.event, - events.properties, - events.distinct_id, - events.elements_chain, + events.event AS event, + events.properties AS properties, + events.distinct_id AS distinct_id, + events.elements_chain AS elements_chain, events.`$session_id` AS session_id, lagInFrame(events.`$session_id`, 1) OVER (PARTITION BY person_id ORDER BY timestamp ASC) AS prev_session_id @@ -294,7 +304,7 @@ LEFT JOIN (SELECT toTimeZone(session_replay_events.start_time, 'UTC') AS start_time, toTimeZone(session_replay_events.end_time, 'UTC') AS end_time, - session_replay_events.session_id + session_replay_events.session_id AS session_id FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, max(session_replay_events.max_last_timestamp) AS end_time, @@ -310,35 +320,37 @@ --- # name: TestSessionsTimelineQueryRunner.test_formal_sessions_for_person ' - SELECT e.uuid, - e.timestamp, - e.event, - e.properties, - e.distinct_id, - e.elements_chain, + SELECT e.uuid AS uuid, + e.timestamp AS timestamp, + e.event AS event, + e.properties AS properties, + e.distinct_id AS distinct_id, + e.elements_chain AS elements_chain, e.session_id AS formal_session_id, first_value(e.uuid) OVER (PARTITION BY tuple(e.person_id, e.session_id_flip_index) ORDER BY toInt64(e.timestamp) ASC RANGE BETWEEN 1800 PRECEDING AND CURRENT ROW) AS informal_session_uuid, dateDiff('s', sre.start_time, sre.end_time) AS recording_duration_s FROM - (SELECT uuid, - person_id, timestamp, event, - properties, - distinct_id, - elements_chain, - session_id, - prev_session_id, - sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) - and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id - ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index + (SELECT uuid AS uuid, + person_id AS person_id, + timestamp AS timestamp, + event AS event, + properties AS properties, + distinct_id AS distinct_id, + elements_chain AS elements_chain, + session_id AS session_id, + prev_session_id AS prev_session_id, + sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) + and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id + ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index FROM - (SELECT events.uuid, + (SELECT events.uuid AS uuid, events__pdi.person_id AS person_id, toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.event, - events.properties, - events.distinct_id, - events.elements_chain, + events.event AS event, + events.properties AS properties, + events.distinct_id AS distinct_id, + events.elements_chain AS elements_chain, events.`$session_id` AS session_id, lagInFrame(events.`$session_id`, 1) OVER (PARTITION BY person_id ORDER BY timestamp ASC) AS prev_session_id @@ -356,7 +368,7 @@ LEFT JOIN (SELECT toTimeZone(session_replay_events.start_time, 'UTC') AS start_time, toTimeZone(session_replay_events.end_time, 'UTC') AS end_time, - session_replay_events.session_id + session_replay_events.session_id AS session_id FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, max(session_replay_events.max_last_timestamp) AS end_time, @@ -372,35 +384,37 @@ --- # name: TestSessionsTimelineQueryRunner.test_formal_sessions_global ' - SELECT e.uuid, - e.timestamp, - e.event, - e.properties, - e.distinct_id, - e.elements_chain, + SELECT e.uuid AS uuid, + e.timestamp AS timestamp, + e.event AS event, + e.properties AS properties, + e.distinct_id AS distinct_id, + e.elements_chain AS elements_chain, e.session_id AS formal_session_id, first_value(e.uuid) OVER (PARTITION BY tuple(e.person_id, e.session_id_flip_index) ORDER BY toInt64(e.timestamp) ASC RANGE BETWEEN 1800 PRECEDING AND CURRENT ROW) AS informal_session_uuid, dateDiff('s', sre.start_time, sre.end_time) AS recording_duration_s FROM - (SELECT uuid, - person_id, timestamp, event, - properties, - distinct_id, - elements_chain, - session_id, - prev_session_id, - sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) - and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id - ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index + (SELECT uuid AS uuid, + person_id AS person_id, + timestamp AS timestamp, + event AS event, + properties AS properties, + distinct_id AS distinct_id, + elements_chain AS elements_chain, + session_id AS session_id, + prev_session_id AS prev_session_id, + sum(if(ifNull(equals(session_id, prev_session_id), isNull(session_id) + and isNull(prev_session_id)), 0, 1)) OVER (PARTITION BY person_id + ORDER BY timestamp ASC ROWS UNBOUNDED PRECEDING) AS session_id_flip_index FROM - (SELECT events.uuid, + (SELECT events.uuid AS uuid, events__pdi.person_id AS person_id, toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.event, - events.properties, - events.distinct_id, - events.elements_chain, + events.event AS event, + events.properties AS properties, + events.distinct_id AS distinct_id, + events.elements_chain AS elements_chain, events.`$session_id` AS session_id, lagInFrame(events.`$session_id`, 1) OVER (PARTITION BY person_id ORDER BY timestamp ASC) AS prev_session_id @@ -418,7 +432,7 @@ LEFT JOIN (SELECT toTimeZone(session_replay_events.start_time, 'UTC') AS start_time, toTimeZone(session_replay_events.end_time, 'UTC') AS end_time, - session_replay_events.session_id + session_replay_events.session_id AS session_id FROM (SELECT min(session_replay_events.min_first_timestamp) AS start_time, max(session_replay_events.max_last_timestamp) AS end_time, diff --git a/posthog/hogql_queries/test/test_hogql_query_runner.py b/posthog/hogql_queries/test/test_hogql_query_runner.py index badc27efef3bf..db744d7d4fa03 100644 --- a/posthog/hogql_queries/test/test_hogql_query_runner.py +++ b/posthog/hogql_queries/test/test_hogql_query_runner.py @@ -17,7 +17,7 @@ class TestHogQLQueryRunner(ClickhouseTestMixin, APIBaseTest): random_uuid: str def _create_random_persons(self) -> str: - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" for index in range(10): _create_person( properties={ diff --git a/posthog/hogql_queries/test/test_persons_query_runner.py b/posthog/hogql_queries/test/test_persons_query_runner.py index 6163293d74f3a..e25bf67166b81 100644 --- a/posthog/hogql_queries/test/test_persons_query_runner.py +++ b/posthog/hogql_queries/test/test_persons_query_runner.py @@ -22,6 +22,7 @@ _create_event, ) from freezegun import freeze_time +from django.test import override_settings class TestPersonsQueryRunner(ClickhouseTestMixin, APIBaseTest): @@ -29,7 +30,7 @@ class TestPersonsQueryRunner(ClickhouseTestMixin, APIBaseTest): random_uuid: str def _create_random_persons(self) -> str: - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" for index in range(10): _create_person( properties={ @@ -72,8 +73,6 @@ def test_default_persons_query(self): ], select_from=ast.JoinExpr(table=ast.Field(chain=["persons"])), where=None, - limit=ast.Constant(value=101), - offset=ast.Constant(value=0), order_by=[ast.OrderExpr(expr=ast.Field(chain=["created_at"]), order="DESC")], ) assert clear_locations(query) == expected @@ -178,7 +177,21 @@ def test_persons_query_limit(self): self.assertEqual(response.results, [[f"jacob7@{self.random_uuid}.posthog.com"]]) self.assertEqual(response.hasMore, True) - def test_source_hogql_query(self): + @override_settings(PERSON_ON_EVENTS_OVERRIDE=True, PERSON_ON_EVENTS_V2_OVERRIDE=True) + def test_source_hogql_query_poe_on(self): + self.random_uuid = self._create_random_persons() + source_query = HogQLQuery(query="SELECT distinct person_id FROM events WHERE event='clicky-4'") + query = PersonsQuery( + select=["properties.email"], + orderBy=["properties.email DESC"], + source=source_query, + ) + runner = self._create_runner(query) + response = runner.calculate() + self.assertEqual(response.results, [[f"jacob4@{self.random_uuid}.posthog.com"]]) + + @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False) + def test_source_hogql_query_poe_off(self): self.random_uuid = self._create_random_persons() source_query = HogQLQuery(query="SELECT distinct person_id FROM events WHERE event='clicky-4'") query = PersonsQuery( diff --git a/posthog/hogql_queries/web_analytics/ctes.py b/posthog/hogql_queries/web_analytics/ctes.py index ab4583e76f0d9..b88cc2d2616d8 100644 --- a/posthog/hogql_queries/web_analytics/ctes.py +++ b/posthog/hogql_queries/web_analytics/ctes.py @@ -4,10 +4,10 @@ PATHNAME_SCROLL_CTE = """ SELECT - events.properties.`$prev_pageview_pathname` AS $pathname, + events.properties.`$prev_pageview_pathname` AS pathname, avg(CASE WHEN toFloat(JSONExtractRaw(events.properties, '$prev_pageview_max_content_percentage')) IS NULL THEN NULL - WHEN toFloat(JSONExtractRaw(events.properties, '$prev_pageview_max_content_percentage')) > 0.8 THEN 100 + WHEN toFloat(JSONExtractRaw(events.properties, '$prev_pageview_max_content_percentage')) > 0.8 THEN 1 ELSE 0 END) AS scroll_gt80_percentage, avg(toFloat(JSONExtractRaw(events.properties, '$prev_pageview_max_scroll_percentage'))) as average_scroll_percentage @@ -16,7 +16,7 @@ WHERE (event = '$pageview' OR event = '$pageleave') AND events.properties.`$prev_pageview_pathname` IS NOT NULL AND ({pathname_scroll_where}) -GROUP BY $pathname +GROUP BY pathname """ COUNTS_CTE = """ diff --git a/posthog/hogql_queries/web_analytics/stats_table.py b/posthog/hogql_queries/web_analytics/stats_table.py index e52d30ca1b791..a7c8c3b4b1f0b 100644 --- a/posthog/hogql_queries/web_analytics/stats_table.py +++ b/posthog/hogql_queries/web_analytics/stats_table.py @@ -4,6 +4,7 @@ from posthog.hogql_queries.web_analytics.ctes import ( COUNTS_CTE, BOUNCE_RATE_CTE, + PATHNAME_SCROLL_CTE, ) from posthog.hogql_queries.web_analytics.web_analytics_query_runner import ( WebAnalyticsQueryRunner, @@ -45,20 +46,36 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: }, backend="cpp", ) - with self.timings.measure("top_pages_query"): - top_sources_query = parse_select( + if self.query.includeScrollDepth: + with self.timings.measure("scroll_depth_query"): + scroll_depth_query = parse_select( + PATHNAME_SCROLL_CTE, + timings=self.timings, + placeholders={ + "pathname_scroll_where": self.events_where(), + "breakdown_by": self.counts_breakdown(), + }, + backend="cpp", + ) + return parse_select( """ SELECT counts.breakdown_value as "context.columns.breakdown_value", counts.total_pageviews as "context.columns.views", counts.unique_visitors as "context.columns.visitors", - bounce_rate.bounce_rate as "context.columns.bounce_rate" + bounce_rate.bounce_rate as "context.columns.bounce_rate", + scroll_depth.average_scroll_percentage as "context.columns.average_scroll_percentage", + scroll_depth.scroll_gt80_percentage as "context.columns.scroll_gt80_percentage" FROM {counts_query} AS counts LEFT OUTER JOIN {bounce_rate_query} AS bounce_rate ON counts.breakdown_value = bounce_rate.breakdown_value +LEFT OUTER JOIN + {scroll_depth_query} AS scroll_depth +ON + counts.breakdown_value = scroll_depth.pathname WHERE {where_breakdown} ORDER BY @@ -70,11 +87,41 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: placeholders={ "counts_query": counts_query, "bounce_rate_query": bounce_rate_query, + "scroll_depth_query": scroll_depth_query, "where_breakdown": self.where_breakdown(), }, backend="cpp", ) - return top_sources_query + else: + with self.timings.measure("stats_table_query"): + return parse_select( + """ + SELECT + counts.breakdown_value as "context.columns.breakdown_value", + counts.total_pageviews as "context.columns.views", + counts.unique_visitors as "context.columns.visitors", + bounce_rate.bounce_rate as "context.columns.bounce_rate" + FROM + {counts_query} AS counts + LEFT OUTER JOIN + {bounce_rate_query} AS bounce_rate + ON + counts.breakdown_value = bounce_rate.breakdown_value + WHERE + {where_breakdown} + ORDER BY + "context.columns.views" DESC, + "context.columns.breakdown_value" DESC + LIMIT 10 + """, + timings=self.timings, + placeholders={ + "counts_query": counts_query, + "bounce_rate_query": bounce_rate_query, + "where_breakdown": self.where_breakdown(), + }, + backend="cpp", + ) def calculate(self): response = execute_hogql_query( diff --git a/posthog/management/commands/create_batch_export_from_app.py b/posthog/management/commands/create_batch_export_from_app.py index b1939656a4fd6..90806ad900fee 100644 --- a/posthog/management/commands/create_batch_export_from_app.py +++ b/posthog/management/commands/create_batch_export_from_app.py @@ -116,7 +116,7 @@ def handle(self, *args, **options): if options.get("backfill_batch_export", False) and dry_run is False: client = sync_connect() - end_at = dt.datetime.utcnow() + end_at = dt.datetime.now(dt.timezone.utc) start_at = end_at - (dt.timedelta(hours=1) if interval == "hour" else dt.timedelta(days=1)) backfill_export( client, diff --git a/posthog/management/commands/test/test_sync_persons_to_clickhouse.py b/posthog/management/commands/test/test_sync_persons_to_clickhouse.py index acde0c4630f19..3609a358054bd 100644 --- a/posthog/management/commands/test/test_sync_persons_to_clickhouse.py +++ b/posthog/management/commands/test/test_sync_persons_to_clickhouse.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from unittest import mock from uuid import UUID, uuid4 @@ -143,7 +143,7 @@ def test_distinct_ids_deleted(self): wraps=posthog.management.commands.sync_persons_to_clickhouse.raw_create_group_ch, ) def test_group_sync(self, mocked_ch_call): - ts = datetime.utcnow() + ts = datetime.now(timezone.utc) Group.objects.create( team_id=self.team.pk, group_type_index=2, @@ -183,12 +183,12 @@ def test_group_sync_updates_group(self, mocked_ch_call): 2, "group-key", {"a": 5}, - timestamp=datetime.utcnow() - timedelta(hours=3), + timestamp=datetime.now(timezone.utc) - timedelta(hours=3), ) group.group_properties = {"a": 5, "b": 3} group.save() - ts_before = datetime.utcnow() + ts_before = datetime.now(timezone.utc) run_group_sync(self.team.pk, live_run=True, sync=True) mocked_ch_call.assert_called_once() @@ -213,7 +213,7 @@ def test_group_sync_updates_group(self, mocked_ch_call): ) self.assertLessEqual( ch_group[4].strftime("%Y-%m-%d %H:%M:%S"), - datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), + datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), ) # second time it's a no-op @@ -225,7 +225,7 @@ def test_group_sync_updates_group(self, mocked_ch_call): wraps=posthog.management.commands.sync_persons_to_clickhouse.raw_create_group_ch, ) def test_group_sync_multiple_entries(self, mocked_ch_call): - ts = datetime.utcnow() + ts = datetime.now(timezone.utc) Group.objects.create( team_id=self.team.pk, group_type_index=2, @@ -430,7 +430,7 @@ def everything_test_run(self, live_run): group_type_index=2, group_key="group-key", group_properties={"a": 1234}, - created_at=datetime.utcnow() - timedelta(hours=3), + created_at=datetime.now(timezone.utc) - timedelta(hours=3), version=5, ) diff --git a/posthog/migrations/0377_flatpersonoverride.py b/posthog/migrations/0377_flatpersonoverride.py new file mode 100644 index 0000000000000..3ba865394fca4 --- /dev/null +++ b/posthog/migrations/0377_flatpersonoverride.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.19 on 2023-12-07 00:38 + +from django.db import migrations, models +import django.db.models.expressions + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0376_externaldataschema_last_synced_at"), + ] + + operations = [ + migrations.CreateModel( + name="FlatPersonOverride", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("team_id", models.BigIntegerField()), + ("old_person_id", models.UUIDField()), + ("override_person_id", models.UUIDField()), + ("oldest_event", models.DateTimeField()), + ("version", models.BigIntegerField(blank=True, null=True)), + ], + ), + migrations.AddIndex( + model_name="flatpersonoverride", + index=models.Index(fields=["team_id", "override_person_id"], name="posthog_fla_team_id_224253_idx"), + ), + migrations.AddConstraint( + model_name="flatpersonoverride", + constraint=models.UniqueConstraint( + fields=("team_id", "old_person_id"), name="flatpersonoverride_unique_old_person_by_team" + ), + ), + migrations.AddConstraint( + model_name="flatpersonoverride", + constraint=models.CheckConstraint( + check=models.Q( + ("old_person_id__exact", django.db.models.expressions.F("override_person_id")), _negated=True + ), + name="flatpersonoverride_check_circular_reference", + ), + ), + ] diff --git a/posthog/migrations/0378_alter_user_theme_mode.py b/posthog/migrations/0378_alter_user_theme_mode.py new file mode 100644 index 0000000000000..fca384ae53435 --- /dev/null +++ b/posthog/migrations/0378_alter_user_theme_mode.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.19 on 2023-12-20 12:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0377_flatpersonoverride"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="theme_mode", + field=models.CharField( + blank=True, + choices=[("light", "Light"), ("dark", "Dark"), ("system", "System")], + max_length=20, + null=True, + ), + ), + ] diff --git a/posthog/migrations/0379_alter_scheduledchange.py b/posthog/migrations/0379_alter_scheduledchange.py new file mode 100644 index 0000000000000..0e0025324151a --- /dev/null +++ b/posthog/migrations/0379_alter_scheduledchange.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.19 on 2023-12-21 14:01 + +from django.db import migrations, models +from django.contrib.postgres.operations import AddIndexConcurrently # type: ignore + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("posthog", "0378_alter_user_theme_mode"), + ] + + operations = [ + migrations.AlterField( + model_name="scheduledchange", + name="record_id", + field=models.CharField(max_length=200), + ), + migrations.AlterField( + model_name="scheduledchange", + name="scheduled_at", + field=models.DateTimeField(), + ), + AddIndexConcurrently( + model_name="scheduledchange", + index=models.Index(fields=["scheduled_at", "executed_at"], name="posthog_sch_schedul_c3687e_idx"), + ), + ] diff --git a/posthog/models/app_metrics/sql.py b/posthog/models/app_metrics/sql.py index fa1a0c856e80d..897e1a15b0d9c 100644 --- a/posthog/models/app_metrics/sql.py +++ b/posthog/models/app_metrics/sql.py @@ -26,6 +26,11 @@ error_details String CODEC(ZSTD(3)) """.strip() +# NOTE: We have producers that take advantage of the timestamp being truncated to the hour, +# i.e. they batch up metrics and send them pre-truncated. If we ever change this truncation +# we need to revisit producers (e.g. the webhook service currently known as rusty-hook or pgqueue). +APP_METRICS_TIMESTAMP_TRUNCATION = "toStartOfHour(timestamp)" + APP_METRICS_DATA_TABLE_SQL = ( lambda: f""" CREATE TABLE IF NOT EXISTS sharded_app_metrics ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}' @@ -35,7 +40,7 @@ ) ENGINE = {SHARDED_APP_METRICS_TABLE_ENGINE()} PARTITION BY toYYYYMM(timestamp) -ORDER BY (team_id, plugin_config_id, job_id, category, toStartOfHour(timestamp), error_type, error_uuid) +ORDER BY (team_id, plugin_config_id, job_id, category, {APP_METRICS_TIMESTAMP_TRUNCATION}, error_type, error_uuid) """ ) diff --git a/posthog/models/feature_flag/feature_flag.py b/posthog/models/feature_flag/feature_flag.py index c339abe44d0ed..80c92452d6d74 100644 --- a/posthog/models/feature_flag/feature_flag.py +++ b/posthog/models/feature_flag/feature_flag.py @@ -297,6 +297,31 @@ def get_cohort_ids( return list(cohort_ids) + def scheduled_changes_dispatcher(self, payload): + from posthog.api.feature_flag import FeatureFlagSerializer + + if "operation" not in payload or "value" not in payload: + raise Exception("Invalid payload") + + context = { + "request": {"user": self.created_by}, + "team_id": self.team_id, + } + serializer_data = {} + + if payload["operation"] == "add_release_condition": + existing_groups = self.get_filters().get("groups", []) + new_groups = payload["value"].get("groups", []) + serializer_data["filters"] = {"groups": existing_groups + new_groups} + elif payload["operation"] == "update_status": + serializer_data["active"] = payload["value"] + else: + raise Exception(f"Unrecognized operation: {payload['operation']}") + + serializer = FeatureFlagSerializer(self, data=serializer_data, context=context, partial=True) + if serializer.is_valid(raise_exception=True): + serializer.save() + @property def uses_cohorts(self) -> bool: for condition in self.conditions: diff --git a/posthog/models/filters/mixins/common.py b/posthog/models/filters/mixins/common.py index 8290a6793ee51..edac192f55547 100644 --- a/posthog/models/filters/mixins/common.py +++ b/posthog/models/filters/mixins/common.py @@ -47,6 +47,7 @@ TREND_FILTER_TYPE_EVENTS, TRENDS_WORLD_MAP, BreakdownAttributionType, + BREAKDOWN_HIDE_OTHER_AGGREGATION, ) from posthog.models.entity import Entity, ExclusionEntity, MathType from posthog.models.filters.mixins.base import BaseParamMixin, BreakdownType @@ -213,6 +214,15 @@ def breakdown_histogram_bin_count(self) -> Optional[int]: pass return None + @cached_property + def breakdown_hide_other_aggregation(self) -> Optional[bool]: + if BREAKDOWN_HIDE_OTHER_AGGREGATION in self._data: + try: + return self._data[BREAKDOWN_HIDE_OTHER_AGGREGATION] in ("True", "true", True) + except ValueError: + pass + return None + @include_dict def breakdown_to_dict(self): result: Dict = {} @@ -228,6 +238,8 @@ def breakdown_to_dict(self): result[BREAKDOWN_ATTRIBUTION_VALUE] = self.breakdown_attribution_value if self.breakdown_histogram_bin_count is not None: result[BREAKDOWN_HISTOGRAM_BIN_COUNT] = self.breakdown_histogram_bin_count + if self.breakdown_hide_other_aggregation is not None: + result[BREAKDOWN_HIDE_OTHER_AGGREGATION] = self.breakdown_hide_other_aggregation if self.breakdown_normalize_url is not None: result[BREAKDOWN_NORMALIZE_URL] = self.breakdown_normalize_url return result diff --git a/posthog/models/person/person.py b/posthog/models/person/person.py index 9ef4a58e18cba..902742219ab0a 100644 --- a/posthog/models/person/person.py +++ b/posthog/models/person/person.py @@ -134,14 +134,6 @@ class Meta: uuid = models.UUIDField() -class PendingPersonOverride(models.Model): - id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID") - team_id = models.BigIntegerField() - old_person_id = models.UUIDField() - override_person_id = models.UUIDField() - oldest_event = models.DateTimeField() - - class PersonOverride(models.Model): """A model of persons to be overriden in merge or merge-like events. @@ -187,6 +179,101 @@ class Meta: version: models.BigIntegerField = models.BigIntegerField(null=True, blank=True) +class PendingPersonOverride(models.Model): + """ + The pending person overrides model/table contains records of merges that + have occurred, but have not yet been integrated into the person overrides + table. + + This table should generally be considered as a log table or queue. When a + merge occurs, it is recorded to the log (added to the queue) as part of the + merge transaction. Later, another process comes along, reading from the + other end of the log (popping from the queue) and applying the necessary + updates to the person overrides table as part of secondary transaction. + + This approach allows us to decouple the set of operations that must occur as + part of an atomic transactional unit during person merging (moving distinct + IDs, merging properties, deleting the subsumed person, etc.) from those that + are more tolerant to eventual consistency (updating person overrides in + Postgres and subsequently relaying those updates to ClickHouse in various + forms to update the person associated with an event.) This decoupling helps + us to minimize the overhead of the primary merge transaction by reducing the + degree of contention within the ingestion pipeline caused by long-running + transactions. This decoupling also allows us to serialize the execution of + all updates to the person overrides table through a single writer, which + allows us to safely update the person overrides table while handling tricky + cases like applying transitive updates without the need for expensive table + constraints to ensure their validity. + """ + + id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID") + team_id = models.BigIntegerField() + old_person_id = models.UUIDField() + override_person_id = models.UUIDField() + oldest_event = models.DateTimeField() + + +class FlatPersonOverride(models.Model): + """ + The (flat) person overrides model/table contains a consolidated record of + all merges that have occurred, but have not yet been integrated into the + ClickHouse events table through a squash operation. Once the effects of a + merge have been integrated into the events table, the associated override + record can be deleted from this table. + + This table is in some sense a materialized view over the pending person + overrides table (i.e. the merge log.) It differs from that base table in + that it should be maintained during updates to account for the effects of + transitive merges. For example, if person A is merged into person B, and + then person B is merged into person C, we'd expect the first record (A->B) + to be updated to reflect that person A has been merged into person C (A->C, + eliding the intermediate step.) + + There are several important expectations about the nature of the data within + this table: + + * A person should only appear as an "old" person at most once for a given + team (as appearing more than once would imply they were merged into + multiple people.) + * A person cannot be merged into themselves (i.e. be both the "old" and + "override" person within a given row.) + * A person should only appear in a table as _either_ an "old" person or + "override" person for a given team -- but never both, as this would + indicate a failure to account for a transitive merge. + + The first two of these expectations can be enforced as constraints, but + unfortunately we've found the third to be too costly to enforce in practice. + Instead, we try to ensure that this invariant holds by serializing all + writes to this table through the ``PendingPersonOverride`` model above. + + The "flat" in the table name is used to distinguish this table from a prior + approach that required multiple tables to maintain the same state but + otherwise has little significance of its own. + """ + + id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID") + team_id = models.BigIntegerField() + old_person_id = models.UUIDField() + override_person_id = models.UUIDField() + oldest_event = models.DateTimeField() + version = models.BigIntegerField(null=True, blank=True) + + class Meta: + indexes = [ + models.Index(fields=["team_id", "override_person_id"]), + ] + constraints = [ + models.UniqueConstraint( + fields=["team_id", "old_person_id"], + name="flatpersonoverride_unique_old_person_by_team", + ), + models.CheckConstraint( + check=~Q(old_person_id__exact=F("override_person_id")), + name="flatpersonoverride_check_circular_reference", + ), + ] + + def get_distinct_ids_for_subquery(person: Person | None, team: Team) -> List[str]: """_summary_ Fetching distinct_ids for a person from CH is slow, so we diff --git a/posthog/models/scheduled_change.py b/posthog/models/scheduled_change.py index 2fea198fd3ba0..ee92cc59c506e 100644 --- a/posthog/models/scheduled_change.py +++ b/posthog/models/scheduled_change.py @@ -1,5 +1,4 @@ from django.db import models -from django.utils import timezone class ScheduledChange(models.Model): @@ -7,10 +6,10 @@ class AllowedModels(models.TextChoices): FEATURE_FLAG = "FeatureFlag", "feature flag" id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID") - record_id = models.IntegerField() + record_id: models.CharField = models.CharField(max_length=200) model_name: models.CharField = models.CharField(max_length=100, choices=AllowedModels.choices) payload: models.JSONField = models.JSONField(default=dict) - scheduled_at: models.DateTimeField = models.DateTimeField(default=timezone.now) + scheduled_at: models.DateTimeField = models.DateTimeField() executed_at: models.DateTimeField = models.DateTimeField(null=True, blank=True) failure_reason = models.CharField(max_length=400, null=True, blank=True) @@ -18,3 +17,8 @@ class AllowedModels(models.TextChoices): created_at: models.DateTimeField = models.DateTimeField(auto_now_add=True) created_by: models.ForeignKey = models.ForeignKey("User", on_delete=models.SET_NULL, null=True) updated_at: models.DateTimeField = models.DateTimeField(auto_now_add=True) + + class Meta: + indexes = [ + models.Index(fields=["scheduled_at", "executed_at"]), + ] diff --git a/posthog/models/user.py b/posthog/models/user.py index 331f0089f72fe..17a7e176a71bc 100644 --- a/posthog/models/user.py +++ b/posthog/models/user.py @@ -118,6 +118,7 @@ def events_column_config_default() -> Dict[str, Any]: class ThemeMode(models.TextChoices): LIGHT = "light", "Light" DARK = "dark", "Dark" + SYSTEM = "system", "System" class User(AbstractUser, UUIDClassicModel): diff --git a/posthog/queries/breakdown_props.py b/posthog/queries/breakdown_props.py index a7a320e8e5dfa..fb9132e83398c 100644 --- a/posthog/queries/breakdown_props.py +++ b/posthog/queries/breakdown_props.py @@ -212,7 +212,7 @@ def get_breakdown_prop_values( **entity_format_params, ) - return insight_sync_execute( + response = insight_sync_execute( elements_query, { "key": filter.breakdown, @@ -233,7 +233,12 @@ def get_breakdown_prop_values( query_type="get_breakdown_prop_values", filter=filter, team_id=team.pk, - )[0][0] + ) + + if filter.using_histogram: + return response[0][0] + else: + return [row[0] for row in response] def _to_value_expression( diff --git a/posthog/queries/funnels/test/__snapshots__/test_breakdowns_by_current_url.ambr b/posthog/queries/funnels/test/__snapshots__/test_breakdowns_by_current_url.ambr index d3b0873234672..d9a0d18dd64b9 100644 --- a/posthog/queries/funnels/test/__snapshots__/test_breakdowns_by_current_url.ambr +++ b/posthog/queries/funnels/test/__snapshots__/test_breakdowns_by_current_url.ambr @@ -1,21 +1,19 @@ # name: TestBreakdownsByCurrentURL.test_breakdown_by_current_url ' - SELECT groupArray(value) - FROM - (SELECT array(if(empty(trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))), '/', trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '')))) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['terminate funnel', 'watched movie'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 100 - OFFSET 0) + SELECT array(if(empty(trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))), '/', trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '')))) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['terminate funnel', 'watched movie'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 100 + OFFSET 0 ' --- # name: TestBreakdownsByCurrentURL.test_breakdown_by_current_url.1 @@ -95,21 +93,19 @@ # name: TestBreakdownsByCurrentURL.test_breakdown_by_pathname ' - SELECT groupArray(value) - FROM - (SELECT array(if(empty(trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))), '/', trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', '')))) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['terminate funnel', 'watched movie'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 100 - OFFSET 0) + SELECT array(if(empty(trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))), '/', trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', '')))) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['terminate funnel', 'watched movie'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 100 + OFFSET 0 ' --- # name: TestBreakdownsByCurrentURL.test_breakdown_by_pathname.1 diff --git a/posthog/queries/funnels/test/__snapshots__/test_funnel.ambr b/posthog/queries/funnels/test/__snapshots__/test_funnel.ambr index 811e4253d9704..c8a3edd3a92a7 100644 --- a/posthog/queries/funnels/test/__snapshots__/test_funnel.ambr +++ b/posthog/queries/funnels/test/__snapshots__/test_funnel.ambr @@ -1012,19 +1012,17 @@ # name: TestFunnelBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen.1 @@ -1103,20 +1101,18 @@ # name: TestFunnelBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'buy' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - AND (has(['xyz'], replaceRegexpAll(JSONExtractRaw(e.properties, '$version'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'buy' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (has(['xyz'], replaceRegexpAll(JSONExtractRaw(e.properties, '$version'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step.1 @@ -1199,19 +1195,17 @@ # name: TestFunnelBreakdown.test_funnel_step_multiple_breakdown_snapshot ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(properties, '$version'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(properties, '$version'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelBreakdown.test_funnel_step_multiple_breakdown_snapshot.1 diff --git a/posthog/queries/funnels/test/__snapshots__/test_funnel_strict.ambr b/posthog/queries/funnels/test/__snapshots__/test_funnel_strict.ambr index d92a99dc53dd8..a7f8077138aee 100644 --- a/posthog/queries/funnels/test/__snapshots__/test_funnel_strict.ambr +++ b/posthog/queries/funnels/test/__snapshots__/test_funnel_strict.ambr @@ -1,19 +1,17 @@ # name: TestFunnelStrictStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelStrictStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen.1 @@ -89,20 +87,18 @@ # name: TestFunnelStrictStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'buy' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - AND (has(['xyz'], replaceRegexpAll(JSONExtractRaw(e.properties, '$version'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'buy' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (has(['xyz'], replaceRegexpAll(JSONExtractRaw(e.properties, '$version'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelStrictStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step.1 @@ -182,19 +178,17 @@ # name: TestFunnelStrictStepsBreakdown.test_funnel_step_multiple_breakdown_snapshot ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(properties, '$version'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(properties, '$version'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelStrictStepsBreakdown.test_funnel_step_multiple_breakdown_snapshot.1 diff --git a/posthog/queries/funnels/test/__snapshots__/test_funnel_unordered.ambr b/posthog/queries/funnels/test/__snapshots__/test_funnel_unordered.ambr index 357ef55a79897..d25d5423a41a7 100644 --- a/posthog/queries/funnels/test/__snapshots__/test_funnel_unordered.ambr +++ b/posthog/queries/funnels/test/__snapshots__/test_funnel_unordered.ambr @@ -1,37 +1,33 @@ # name: TestFunnelUnorderedStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelUnorderedStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen.1 ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelUnorderedStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen.2 @@ -154,37 +150,33 @@ # name: TestFunnelUnorderedStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelUnorderedStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step.1 ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelUnorderedStepsBreakdown.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step.2 @@ -315,37 +307,33 @@ # name: TestFunnelUnorderedStepsBreakdown.test_funnel_step_multiple_breakdown_snapshot ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(properties, '$version'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(properties, '$version'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelUnorderedStepsBreakdown.test_funnel_step_multiple_breakdown_snapshot.1 ' - SELECT groupArray(value) - FROM - (SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(properties, '$version'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event IN ['buy', 'sign up'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(properties, '$version'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event IN ['buy', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFunnelUnorderedStepsBreakdown.test_funnel_step_multiple_breakdown_snapshot.2 diff --git a/posthog/queries/test/__snapshots__/test_trends.ambr b/posthog/queries/test/__snapshots__/test_trends.ambr index a11d39b9ada8f..90570288abc8b 100644 --- a/posthog/queries/test/__snapshots__/test_trends.ambr +++ b/posthog/queries/test/__snapshots__/test_trends.ambr @@ -152,27 +152,25 @@ # name: TestTrends.test_breakdown_by_group_props_person_on_events ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_by_group_props_person_on_events.1 @@ -204,7 +202,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''), (['finance', 'technology']), (['finance', 'technology']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['finance', 'technology']), (['finance', 'technology']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e LEFT JOIN (SELECT group_key, @@ -263,28 +261,26 @@ # name: TestTrends.test_breakdown_by_group_props_with_person_filter_person_on_events ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - AND (has(['value'], replaceRegexpAll(JSONExtractRaw(e.person_properties, 'key'), '^"|"$', ''))) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND (has(['value'], replaceRegexpAll(JSONExtractRaw(e.person_properties, 'key'), '^"|"$', ''))) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_by_group_props_with_person_filter_person_on_events.1 @@ -316,7 +312,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''), (['finance']), (['finance']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['finance']), (['finance']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e LEFT JOIN (SELECT group_key, @@ -345,22 +341,20 @@ # name: TestTrends.test_breakdown_filtering_with_properties_in_new_format ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-22 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') - AND (((has(['Firefox'], replaceRegexpAll(JSONExtractRaw(e.properties, '$browser'), '^"|"$', ''))) - OR (has(['Windows'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) - AND (has(['Mac'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-22 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') + AND (((has(['Firefox'], replaceRegexpAll(JSONExtractRaw(e.properties, '$browser'), '^"|"$', ''))) + OR (has(['Windows'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) + AND (has(['Mac'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_filtering_with_properties_in_new_format.1 @@ -392,7 +386,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''), (['second url']), (['second url']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['second url']), (['second url']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'sign up' @@ -414,22 +408,20 @@ # name: TestTrends.test_breakdown_filtering_with_properties_in_new_format.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-22 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') - AND (((has(['Firefox'], replaceRegexpAll(JSONExtractRaw(e.properties, '$browser'), '^"|"$', ''))) - AND (has(['Windows'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) - AND (has(['Mac'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-22 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') + AND (((has(['Firefox'], replaceRegexpAll(JSONExtractRaw(e.properties, '$browser'), '^"|"$', ''))) + AND (has(['Windows'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) + AND (has(['Mac'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_filtering_with_properties_in_new_format.3 @@ -443,33 +435,31 @@ # name: TestTrends.test_breakdown_weekly_active_users_aggregated ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, - count(DISTINCT pdi.person_id) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, + count(DISTINCT pdi.person_id) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-11 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-11 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-11 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-11 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_weekly_active_users_aggregated.1 ' SELECT count(DISTINCT pdi.person_id) AS total, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), (['val', 'bor']), (['val', 'bor']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['val', 'bor']), (['val', 'bor']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events AS e INNER JOIN (SELECT distinct_id, @@ -491,33 +481,31 @@ # name: TestTrends.test_breakdown_weekly_active_users_aggregated_materialized ' - SELECT groupArray(value) - FROM - (SELECT "mat_key" AS value, - count(DISTINCT pdi.person_id) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + SELECT "mat_key" AS value, + count(DISTINCT pdi.person_id) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = '$pageview' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-11 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-11 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-11 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-11 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_weekly_active_users_aggregated_materialized.1 ' SELECT count(DISTINCT pdi.person_id) AS total, - transform("mat_key", (['val', 'bor']), (['val', 'bor']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf("mat_key", ''), '$$_posthog_breakdown_null_$$'), (['val', 'bor']), (['val', 'bor']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events AS e INNER JOIN (SELECT distinct_id, @@ -539,46 +527,44 @@ # name: TestTrends.test_breakdown_weekly_active_users_daily_based_on_action ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, - count(DISTINCT pdi.person_id) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, + count(DISTINCT pdi.person_id) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person WHERE team_id = 2 - AND ((event = '$pageview' - AND (has(['p1', 'p2', 'p3'], replaceRegexpAll(JSONExtractRaw(person_props, 'name'), '^"|"$', '')) - AND pdi.person_id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND ((has(['p1', 'p2'], replaceRegexpAll(JSONExtractRaw(properties, 'name'), '^"|"$', '')))) ) - GROUP BY id - HAVING max(is_deleted) = 0 - AND ((has(['p1', 'p2'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'name'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1)))) - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND ((event = '$pageview' + AND (has(['p1', 'p2', 'p3'], replaceRegexpAll(JSONExtractRaw(person_props, 'name'), '^"|"$', '')) + AND pdi.person_id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND ((has(['p1', 'p2'], replaceRegexpAll(JSONExtractRaw(properties, 'name'), '^"|"$', '')))) ) + GROUP BY id + HAVING max(is_deleted) = 0 + AND ((has(['p1', 'p2'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'name'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1)))) + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_weekly_active_users_daily_based_on_action.1 @@ -625,7 +611,7 @@ CROSS JOIN (SELECT toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) AS timestamp, pdi.person_id AS person_id, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), (['val']), (['val']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['val']), (['val']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -679,27 +665,25 @@ # name: TestTrends.test_breakdown_with_filter_groups_person_on_events ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - AND (has(['finance'], replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND (has(['finance'], replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_with_filter_groups_person_on_events.1 @@ -731,7 +715,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), (['uh', 'oh']), (['uh', 'oh']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['uh', 'oh']), (['uh', 'oh']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e LEFT JOIN (SELECT group_key, @@ -772,34 +756,32 @@ # name: TestTrends.test_breakdown_with_filter_groups_person_on_events_v2.1 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT OUTER JOIN - (SELECT argMax(override_person_id, version) as person_id, - old_person_id - FROM person_overrides - WHERE team_id = 2 - GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id - LEFT JOIN - (SELECT group_key, - argMax(group_properties, _timestamp) AS group_properties_0 - FROM groups - WHERE team_id = 2 - AND group_type_index = 0 - GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT OUTER JOIN + (SELECT argMax(override_person_id, version) as person_id, + old_person_id + FROM person_overrides WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - AND (has(['finance'], replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''))) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND (has(['finance'], replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''))) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_breakdown_with_filter_groups_person_on_events_v2.2 @@ -831,7 +813,7 @@ day_start UNION ALL SELECT count(DISTINCT if(notEmpty(overrides.person_id), overrides.person_id, e.person_id)) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), (['uh', 'oh']), (['uh', 'oh']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['uh', 'oh']), (['uh', 'oh']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e LEFT OUTER JOIN (SELECT argMax(override_person_id, version) as person_id, @@ -866,19 +848,17 @@ # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - count(*) as count - FROM events e SAMPLE 1.0 - WHERE team_id = 2 - AND ((event = 'sign up')) - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + count(*) as count + FROM events e SAMPLE 1.0 + WHERE team_id = 2 + AND ((event = 'sign up')) + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.1 @@ -904,13 +884,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['other_value', '', 'value'] as breakdown_value) ARRAY + (SELECT ['other_value', '$$_posthog_breakdown_null_$$', 'value'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT count(DISTINCT pdi.person_id) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['other_value', '', 'value']), (['other_value', '', 'value']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['other_value', '$$_posthog_breakdown_null_$$', 'value']), (['other_value', '$$_posthog_breakdown_null_$$', 'value']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e SAMPLE 1.0 INNER JOIN (SELECT distinct_id, @@ -936,19 +916,17 @@ # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - count(*) as count - FROM events e SAMPLE 1.0 - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + count(*) as count + FROM events e SAMPLE 1.0 + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.3 @@ -974,13 +952,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['other_value', '', 'value'] as breakdown_value) ARRAY + (SELECT ['other_value', '$$_posthog_breakdown_null_$$', 'value'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT count(DISTINCT pdi.person_id) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['other_value', '', 'value']), (['other_value', '', 'value']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['other_value', '$$_posthog_breakdown_null_$$', 'value']), (['other_value', '$$_posthog_breakdown_null_$$', 'value']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e SAMPLE 1.0 INNER JOIN (SELECT distinct_id, @@ -1237,39 +1215,37 @@ # name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', '') AS value, - count(DISTINCT pdi.person_id) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - AND id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND (has(['filter_val'], replaceRegexpAll(JSONExtractRaw(properties, 'filter_prop'), '^"|"$', ''))) ) - GROUP BY id - HAVING max(is_deleted) = 0 - AND (has(['filter_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'filter_prop'), '^"|"$', ''))) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', '') AS value, + count(DISTINCT pdi.person_id) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person + WHERE team_id = 2 + AND id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND (has(['filter_val'], replaceRegexpAll(JSONExtractRaw(properties, 'filter_prop'), '^"|"$', ''))) ) + GROUP BY id + HAVING max(is_deleted) = 0 + AND (has(['filter_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'filter_prop'), '^"|"$', ''))) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter.1 @@ -1316,7 +1292,7 @@ CROSS JOIN (SELECT toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) AS timestamp, pdi.person_id AS person_id, - transform(replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', ''), (['some_val2', 'some_val']), (['some_val2', 'some_val']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['some_val2', 'some_val']), (['some_val2', 'some_val']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -1363,27 +1339,25 @@ # name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter_poe_v2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_properties, '$some_prop'), '^"|"$', '') AS value, - count(DISTINCT if(notEmpty(overrides.person_id), overrides.person_id, e.person_id)) as count - FROM events e - LEFT OUTER JOIN - (SELECT argMax(override_person_id, version) as person_id, - old_person_id - FROM person_overrides - WHERE team_id = 2 - GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + SELECT replaceRegexpAll(JSONExtractRaw(person_properties, '$some_prop'), '^"|"$', '') AS value, + count(DISTINCT if(notEmpty(overrides.person_id), overrides.person_id, e.person_id)) as count + FROM events e + LEFT OUTER JOIN + (SELECT argMax(override_person_id, version) as person_id, + old_person_id + FROM person_overrides WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - AND (has(['filter_val'], replaceRegexpAll(JSONExtractRaw(e.person_properties, 'filter_prop'), '^"|"$', ''))) - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + AND (has(['filter_val'], replaceRegexpAll(JSONExtractRaw(e.person_properties, 'filter_prop'), '^"|"$', ''))) + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter_poe_v2.1 @@ -1430,7 +1404,7 @@ CROSS JOIN (SELECT toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) AS timestamp, if(notEmpty(overrides.person_id), overrides.person_id, e.person_id) AS person_id, - transform(replaceRegexpAll(JSONExtractRaw(person_properties, '$some_prop'), '^"|"$', ''), (['some_val2', 'some_val']), (['some_val2', 'some_val']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(person_properties, '$some_prop'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['some_val2', 'some_val']), (['some_val2', 'some_val']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e LEFT OUTER JOIN (SELECT argMax(override_person_id, version) as person_id, @@ -1492,38 +1466,36 @@ # name: TestTrends.test_person_filtering_in_cohort_in_action ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND ((event = 'sign up' - AND (pdi.person_id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', '')))) ) - GROUP BY id - HAVING max(is_deleted) = 0 - AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1)))) - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + WHERE team_id = 2 + AND ((event = 'sign up' + AND (pdi.person_id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', '')))) ) + GROUP BY id + HAVING max(is_deleted) = 0 + AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1)))) + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_person_filtering_in_cohort_in_action.1 @@ -1549,13 +1521,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['', 'value', 'other_value'] as breakdown_value) ARRAY + (SELECT ['$$_posthog_breakdown_null_$$', 'value', 'other_value'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['', 'value', 'other_value']), (['', 'value', 'other_value']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'value', 'other_value']), (['$$_posthog_breakdown_null_$$', 'value', 'other_value']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -1593,38 +1565,36 @@ # name: TestTrends.test_person_filtering_in_cohort_in_action_poe_v2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT OUTER JOIN - (SELECT argMax(override_person_id, version) as person_id, - old_person_id - FROM person_overrides - WHERE team_id = 2 - GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT OUTER JOIN + (SELECT argMax(override_person_id, version) as person_id, + old_person_id + FROM person_overrides WHERE team_id = 2 - AND ((event = 'sign up' - AND (if(notEmpty(overrides.person_id), overrides.person_id, e.person_id) IN - (SELECT id - FROM person - WHERE team_id = 2 - AND id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', '')))) ) - GROUP BY id - HAVING max(is_deleted) = 0 - AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1)))) - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + WHERE team_id = 2 + AND ((event = 'sign up' + AND (if(notEmpty(overrides.person_id), overrides.person_id, e.person_id) IN + (SELECT id + FROM person + WHERE team_id = 2 + AND id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', '')))) ) + GROUP BY id + HAVING max(is_deleted) = 0 + AND ((has(['some_val'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1)))) + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_person_filtering_in_cohort_in_action_poe_v2.1 @@ -1650,13 +1620,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['', 'value', 'other_value'] as breakdown_value) ARRAY + (SELECT ['$$_posthog_breakdown_null_$$', 'value', 'other_value'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['', 'value', 'other_value']), (['', 'value', 'other_value']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'value', 'other_value']), (['$$_posthog_breakdown_null_$$', 'value', 'other_value']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e LEFT OUTER JOIN (SELECT argMax(override_person_id, version) as person_id, @@ -2278,19 +2248,17 @@ # name: TestTrends.test_timezones_daily.4 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-29 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-29 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_timezones_daily.5 @@ -2322,7 +2290,7 @@ day_start UNION ALL SELECT count(DISTINCT pdi.person_id) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''), (['Mac']), (['Mac']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['Mac']), (['Mac']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -2482,19 +2450,17 @@ # name: TestTrends.test_timezones_daily_minus_utc.4 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'America/Phoenix') >= toDateTime('2019-12-29 00:00:00', 'America/Phoenix') - AND toTimeZone(timestamp, 'America/Phoenix') <= toDateTime('2020-01-05 23:59:59', 'America/Phoenix') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'America/Phoenix') >= toDateTime('2019-12-29 00:00:00', 'America/Phoenix') + AND toTimeZone(timestamp, 'America/Phoenix') <= toDateTime('2020-01-05 23:59:59', 'America/Phoenix') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_timezones_daily_minus_utc.5 @@ -2526,7 +2492,7 @@ day_start UNION ALL SELECT count(DISTINCT pdi.person_id) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'America/Phoenix')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''), (['Mac']), (['Mac']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['Mac']), (['Mac']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -2686,19 +2652,17 @@ # name: TestTrends.test_timezones_daily_plus_utc.4 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'Asia/Tokyo') >= toDateTime('2019-12-29 00:00:00', 'Asia/Tokyo') - AND toTimeZone(timestamp, 'Asia/Tokyo') <= toDateTime('2020-01-05 23:59:59', 'Asia/Tokyo') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'Asia/Tokyo') >= toDateTime('2019-12-29 00:00:00', 'Asia/Tokyo') + AND toTimeZone(timestamp, 'Asia/Tokyo') <= toDateTime('2020-01-05 23:59:59', 'Asia/Tokyo') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_timezones_daily_plus_utc.5 @@ -2730,7 +2694,7 @@ day_start UNION ALL SELECT count(DISTINCT pdi.person_id) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'Asia/Tokyo')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''), (['Mac']), (['Mac']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['Mac']), (['Mac']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -3300,43 +3264,41 @@ # name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - AND id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND (((has(['android'], replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''))) - OR (has(['safari'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''))))) ) - GROUP BY id - HAVING max(is_deleted) = 0 - AND (((has(['android'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$os'), '^"|"$', ''))) - OR (has(['safari'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$browser'), '^"|"$', ''))))) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-07-01 23:59:59', 'UTC') - AND (((NOT (replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') ILIKE '%@posthog.com%')) - OR (has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', ''))))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person + WHERE team_id = 2 + AND id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND (((has(['android'], replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''))) + OR (has(['safari'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''))))) ) + GROUP BY id + HAVING max(is_deleted) = 0 + AND (((has(['android'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$os'), '^"|"$', ''))) + OR (has(['safari'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$browser'), '^"|"$', ''))))) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-07-01 23:59:59', 'UTC') + AND (((NOT (replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') ILIKE '%@posthog.com%')) + OR (has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', ''))))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns.1 @@ -3368,7 +3330,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', ''), (['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com']), (['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com']), (['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -3411,44 +3373,42 @@ # name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - AND id IN - (SELECT id - FROM person - WHERE team_id = 2 - AND ((((has(['android'], replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''))) - AND (has(['chrome'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''))))) - AND (replaceRegexpAll(JSONExtractRaw(properties, 'email'), '^"|"$', '') ILIKE '%@posthog.com%')) ) - GROUP BY id - HAVING max(is_deleted) = 0 - AND ((((has(['android'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$os'), '^"|"$', ''))) - AND (has(['chrome'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$browser'), '^"|"$', ''))))) - AND (replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'email'), '^"|"$', '') ILIKE '%@posthog.com%')) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-07-01 23:59:59', 'UTC') - AND ((has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', '')))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person + WHERE team_id = 2 + AND id IN + (SELECT id + FROM person + WHERE team_id = 2 + AND ((((has(['android'], replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', ''))) + AND (has(['chrome'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', ''))))) + AND (replaceRegexpAll(JSONExtractRaw(properties, 'email'), '^"|"$', '') ILIKE '%@posthog.com%')) ) + GROUP BY id + HAVING max(is_deleted) = 0 + AND ((((has(['android'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$os'), '^"|"$', ''))) + AND (has(['chrome'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$browser'), '^"|"$', ''))))) + AND (replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), 'email'), '^"|"$', '') ILIKE '%@posthog.com%')) SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-07-01 23:59:59', 'UTC') + AND ((has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', '')))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns.3 @@ -3480,7 +3440,7 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', ''), (['test2@posthog.com']), (['test2@posthog.com']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['test2@posthog.com']), (['test2@posthog.com']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -3595,33 +3555,31 @@ # name: TestTrends.test_trends_aggregate_by_distinct_id.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', '') AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', '') AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-24 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2019-12-31 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person + WHERE team_id = 2 + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-24 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2019-12-31 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_aggregate_by_distinct_id.3 @@ -3647,13 +3605,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['some_val', ''] as breakdown_value) ARRAY + (SELECT ['some_val', '$$_posthog_breakdown_null_$$'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT count(DISTINCT e.distinct_id) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', ''), (['some_val', '']), (['some_val', '']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['some_val', '$$_posthog_breakdown_null_$$']), (['some_val', '$$_posthog_breakdown_null_$$']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -3770,19 +3728,17 @@ # name: TestTrends.test_trends_aggregate_by_distinct_id.6 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-24 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2019-12-31 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-24 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2019-12-31 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_aggregate_by_distinct_id.7 @@ -3808,13 +3764,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT [''] as breakdown_value) ARRAY + (SELECT ['$$_posthog_breakdown_null_$$'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT count(DISTINCT e.distinct_id) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', ''), (['']), (['']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$']), (['$$_posthog_breakdown_null_$$']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'sign up' @@ -3885,19 +3841,17 @@ # name: TestTrends.test_trends_breakdown_cumulative ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_breakdown_cumulative.1 @@ -3923,7 +3877,7 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['', 'value', 'other_value'] as breakdown_value) ARRAY + (SELECT ['$$_posthog_breakdown_null_$$', 'value', 'other_value'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start @@ -3935,7 +3889,7 @@ min(timestamp) as timestamp, breakdown_value FROM - (SELECT pdi.person_id as person_id, timestamp, transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['', 'value', 'other_value']), (['', 'value', 'other_value']), '$$_posthog_breakdown_other_$$') as breakdown_value + (SELECT pdi.person_id as person_id, timestamp, transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'value', 'other_value']), (['$$_posthog_breakdown_null_$$', 'value', 'other_value']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -3963,26 +3917,24 @@ # name: TestTrends.test_trends_breakdown_cumulative_poe_v2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - count(*) as count - FROM events e - LEFT OUTER JOIN - (SELECT argMax(override_person_id, version) as person_id, - old_person_id - FROM person_overrides - WHERE team_id = 2 - GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT OUTER JOIN + (SELECT argMax(override_person_id, version) as person_id, + old_person_id + FROM person_overrides WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - AND notEmpty(e.person_id) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY old_person_id) AS overrides ON e.person_id = overrides.old_person_id + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + AND notEmpty(e.person_id) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_breakdown_cumulative_poe_v2.1 @@ -4008,7 +3960,7 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['', 'value', 'other_value'] as breakdown_value) ARRAY + (SELECT ['$$_posthog_breakdown_null_$$', 'value', 'other_value'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start @@ -4020,7 +3972,7 @@ min(timestamp) as timestamp, breakdown_value FROM - (SELECT if(notEmpty(overrides.person_id), overrides.person_id, e.person_id) as person_id, timestamp, transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['', 'value', 'other_value']), (['', 'value', 'other_value']), '$$_posthog_breakdown_other_$$') as breakdown_value + (SELECT if(notEmpty(overrides.person_id), overrides.person_id, e.person_id) as person_id, timestamp, transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'value', 'other_value']), (['$$_posthog_breakdown_null_$$', 'value', 'other_value']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e LEFT OUTER JOIN (SELECT argMax(override_person_id, version) as person_id, @@ -4049,28 +4001,26 @@ # name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - quantile(0.50)(session_duration) as count - FROM events e - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + quantile(0.50)(session_duration) as count + FROM events e + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown.1 @@ -4084,7 +4034,7 @@ FROM (SELECT sessions.$session_id, session_duration, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['value2', 'value1', '']), (['value2', 'value1', '']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['value2', 'value1', '$$_posthog_breakdown_null_$$']), (['value2', 'value1', '$$_posthog_breakdown_null_$$']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e INNER JOIN (SELECT "$session_id", @@ -4108,28 +4058,26 @@ # name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - quantile(0.50)(session_duration) as count - FROM events e - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + quantile(0.50)(session_duration) as count + FROM events e + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown.3 @@ -4143,7 +4091,7 @@ FROM (SELECT sessions.$session_id, session_duration, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['value2', 'value1', '']), (['value2', 'value1', '']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['value2', 'value1', '$$_posthog_breakdown_null_$$']), (['value2', 'value1', '$$_posthog_breakdown_null_$$']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e INNER JOIN (SELECT "$session_id", @@ -4338,32 +4286,30 @@ # name: TestTrends.test_trends_count_per_user_average_aggregated_with_event_property_breakdown_with_sampling ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'color'), '^"|"$', '') AS value, - count(*) as count - FROM events e SAMPLE 1.0 - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'color'), '^"|"$', '') AS value, + count(*) as count + FROM events e SAMPLE 1.0 + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = 'viewed video' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-07 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id + FROM person + WHERE team_id = 2 + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = 'viewed video' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-07 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_count_per_user_average_aggregated_with_event_property_breakdown_with_sampling.1 @@ -4374,7 +4320,7 @@ FROM (SELECT COUNT(*) AS intermediate_count, pdi.person_id, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'color'), '^"|"$', ''), (['red', 'blue', '']), (['red', 'blue', '']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'color'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['red', 'blue', '$$_posthog_breakdown_null_$$']), (['red', 'blue', '$$_posthog_breakdown_null_$$']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events AS e SAMPLE 1.0 INNER JOIN (SELECT distinct_id, @@ -4538,42 +4484,40 @@ # name: TestTrends.test_trends_person_breakdown_with_session_property_single_aggregate_math_and_breakdown ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', '') AS value, - quantile(0.50)(session_duration) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + SELECT replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', '') AS value, + quantile(0.50)(session_duration) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person + WHERE team_id = 2 + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_person_breakdown_with_session_property_single_aggregate_math_and_breakdown.1 @@ -4587,7 +4531,7 @@ FROM (SELECT sessions.$session_id, session_duration, - transform(replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', ''), (['some_val', 'another_val']), (['some_val', 'another_val']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(person_props, '$some_prop'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['some_val', 'another_val']), (['some_val', 'another_val']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -4773,28 +4717,26 @@ # name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - quantile(0.50)(session_duration) as count - FROM events e - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + quantile(0.50)(session_duration) as count + FROM events e + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns.1 @@ -4835,7 +4777,7 @@ (SELECT sessions.$session_id, session_duration, toStartOfWeek(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC'), 0) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['value2', 'value1']), (['value2', 'value1']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['value2', 'value1']), (['value2', 'value1']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events AS e INNER JOIN (SELECT "$session_id", @@ -4866,28 +4808,26 @@ # name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns.2 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, - quantile(0.50)(session_duration) as count - FROM events e - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" - WHERE team_id = 2 - AND event = 'sign up' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') AS value, + quantile(0.50)(session_duration) as count + FROM events e + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = 'sign up' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns.3 @@ -4928,7 +4868,7 @@ (SELECT sessions.$session_id, session_duration, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), (['value2', 'value1']), (['value2', 'value1']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['value2', 'value1']), (['value2', 'value1']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events AS e INNER JOIN (SELECT "$session_id", diff --git a/posthog/queries/test/test_trends.py b/posthog/queries/test/test_trends.py index 43872175c5b84..16563da5a583d 100644 --- a/posthog/queries/test/test_trends.py +++ b/posthog/queries/test/test_trends.py @@ -38,6 +38,7 @@ set_instance_setting, ) from posthog.models.person.util import create_person_distinct_id +from posthog.queries.trends.breakdown import BREAKDOWN_OTHER_STRING_LABEL from posthog.queries.trends.trends import Trends from posthog.test.base import ( APIBaseTest, @@ -1323,7 +1324,7 @@ def test_trends_breakdown_with_session_property_single_aggregate_math_and_breakd # empty has: 1 seconds self.assertEqual( [resp["breakdown_value"] for resp in daily_response], - ["value2", "value1", ""], + ["value2", "value1", "$$_posthog_breakdown_null_$$"], ) self.assertEqual([resp["aggregated_value"] for resp in daily_response], [12.5, 10, 1]) @@ -4737,7 +4738,7 @@ def test_breakdown_filtering_limit(self): ) assert len(response) == 26 assert response[0]["label"] == "Other" - assert response[0]["breakdown_value"] == "$$_posthog_breakdown_other_$$" + assert response[0]["breakdown_value"] == BREAKDOWN_OTHER_STRING_LABEL response = Trends().run( Filter( @@ -4760,7 +4761,7 @@ def test_breakdown_filtering_limit(self): ) assert len(response) == 51 assert response[0]["label"] == "Other" - assert response[0]["breakdown_value"] == "$$_posthog_breakdown_other_$$" + assert response[0]["breakdown_value"] == BREAKDOWN_OTHER_STRING_LABEL response = Trends().run( Filter( @@ -7629,7 +7630,7 @@ def test_trends_count_per_user_average_with_event_property_breakdown(self): assert len(daily_response) == 3 assert daily_response[0]["breakdown_value"] == "red" assert daily_response[1]["breakdown_value"] == "blue" - assert daily_response[2]["breakdown_value"] == "" + assert daily_response[2]["breakdown_value"] == "$$_posthog_breakdown_null_$$" assert daily_response[0]["days"] == [ "2020-01-01", "2020-01-02", @@ -7699,7 +7700,7 @@ def test_trends_count_per_user_average_aggregated_with_event_property_breakdown( assert len(daily_response) == 3 assert daily_response[0]["breakdown_value"] == "red" assert daily_response[1]["breakdown_value"] == "blue" - assert daily_response[2]["breakdown_value"] == "" + assert daily_response[2]["breakdown_value"] == "$$_posthog_breakdown_null_$$" assert daily_response[0]["aggregated_value"] == 2.0 # red assert daily_response[1]["aggregated_value"] == 1.0 # blue assert daily_response[2]["aggregated_value"] == 1.0 # none @@ -7726,7 +7727,7 @@ def test_trends_count_per_user_average_aggregated_with_event_property_breakdown_ assert len(daily_response) == 3 assert daily_response[0]["breakdown_value"] == "red" assert daily_response[1]["breakdown_value"] == "blue" - assert daily_response[2]["breakdown_value"] == "" + assert daily_response[2]["breakdown_value"] == "$$_posthog_breakdown_null_$$" assert daily_response[0]["aggregated_value"] == 2.0 # red assert daily_response[1]["aggregated_value"] == 1.0 # blue assert daily_response[2]["aggregated_value"] == 1.0 # none diff --git a/posthog/queries/trends/breakdown.py b/posthog/queries/trends/breakdown.py index 51f27d6bf1e0b..8d454c8081179 100644 --- a/posthog/queries/trends/breakdown.py +++ b/posthog/queries/trends/breakdown.py @@ -58,6 +58,7 @@ SESSION_DURATION_BREAKDOWN_INNER_SQL, VOLUME_PER_ACTOR_BREAKDOWN_AGGREGATE_SQL, VOLUME_PER_ACTOR_BREAKDOWN_INNER_SQL, + BREAKDOWN_PROP_JOIN_WITH_OTHER_SQL, ) from posthog.queries.trends.util import ( COUNT_PER_ACTOR_MATH_FUNCTIONS, @@ -83,6 +84,8 @@ BREAKDOWN_OTHER_STRING_LABEL = "$$_posthog_breakdown_other_$$" BREAKDOWN_OTHER_NUMERIC_LABEL = 9007199254740991 # pow(2, 53) - 1, for JS compatibility +BREAKDOWN_NULL_STRING_LABEL = "$$_posthog_breakdown_null_$$" +BREAKDOWN_NULL_NUMERIC_LABEL = 9007199254740990 # pow(2, 53) - 2, for JS compatibility class TrendsBreakdown: @@ -452,6 +455,7 @@ def _breakdown_prop_params(self, aggregate_operation: str, math_params: Dict): breakdown_value = self._get_breakdown_value(self.filter.breakdown) breakdown_other_value: str | int = BREAKDOWN_OTHER_STRING_LABEL + breakdown_null_value: str | int = BREAKDOWN_NULL_STRING_LABEL numeric_property_filter = "" if self.filter.using_histogram: numeric_property_filter = f"AND {breakdown_value} is not null" @@ -461,19 +465,36 @@ def _breakdown_prop_params(self, aggregate_operation: str, math_params: Dict): # Not adding "Other" for the custom session duration filter. pass else: - all_values_are_numeric = all(isinstance(value, int) or isinstance(value, float) for value in values_arr) - all_values_are_string = all(isinstance(value, str) for value in values_arr) + all_values_are_numeric_or_none = all( + isinstance(value, int) or isinstance(value, float) or value is None for value in values_arr + ) + all_values_are_string_or_none = all(isinstance(value, str) or value is None for value in values_arr) - if all_values_are_numeric: + if all_values_are_numeric_or_none: breakdown_other_value = BREAKDOWN_OTHER_NUMERIC_LABEL - elif not all_values_are_string: - breakdown_value = f"toString({breakdown_value})" + breakdown_null_value = BREAKDOWN_NULL_NUMERIC_LABEL + values_arr = [BREAKDOWN_NULL_NUMERIC_LABEL if value is None else value for value in values_arr] + else: + if not all_values_are_string_or_none: + breakdown_value = f"toString({breakdown_value})" + breakdown_value = f"nullIf({breakdown_value}, '')" + values_arr = [BREAKDOWN_NULL_STRING_LABEL if value in (None, "") else value for value in values_arr] + breakdown_value = f"transform(ifNull({breakdown_value}, %(breakdown_null_value)s), (%(values)s), (%(values)s), %(breakdown_other_value)s)" - breakdown_value = f"transform({breakdown_value}, (%(values)s), (%(values)s), %(other_value)s)" + if self.filter.using_histogram: + sql_query = BREAKDOWN_HISTOGRAM_PROP_JOIN_SQL + elif self.filter.breakdown_hide_other_aggregation: + sql_query = BREAKDOWN_PROP_JOIN_SQL + else: + sql_query = BREAKDOWN_PROP_JOIN_WITH_OTHER_SQL return ( - {"values": values_arr, "other_value": breakdown_other_value}, - BREAKDOWN_PROP_JOIN_SQL if not self.filter.using_histogram else BREAKDOWN_HISTOGRAM_PROP_JOIN_SQL, + { + "values": values_arr, + "breakdown_other_value": breakdown_other_value, + "breakdown_null_value": breakdown_null_value, + }, + sql_query, { "breakdown_value_expr": breakdown_value, "numeric_property_filter": numeric_property_filter, @@ -713,8 +734,10 @@ def _determine_breakdown_label( ) -> str: if breakdown_type == "cohort": return get_breakdown_cohort_name(breakdown_value) - elif str(value) == "$$_posthog_breakdown_other_$$": + elif str(value) == BREAKDOWN_OTHER_STRING_LABEL or value == BREAKDOWN_OTHER_NUMERIC_LABEL: return "Other" + elif str(value) == BREAKDOWN_NULL_STRING_LABEL or value == BREAKDOWN_NULL_NUMERIC_LABEL: + return "none" else: return str(value) or "none" diff --git a/posthog/queries/trends/sql.py b/posthog/queries/trends/sql.py index de597ad89d98f..be4a56cc1debd 100644 --- a/posthog/queries/trends/sql.py +++ b/posthog/queries/trends/sql.py @@ -98,7 +98,6 @@ """ TOP_ELEMENTS_ARRAY_OF_KEY_SQL = """ -SELECT groupArray(value) FROM ( SELECT {value_expression}, {aggregate_operation} as count @@ -112,7 +111,6 @@ GROUP BY value ORDER BY count DESC, value DESC LIMIT %(limit)s OFFSET %(offset)s -) """ HISTOGRAM_ELEMENTS_ARRAY_OF_KEY_SQL = """ @@ -372,8 +370,14 @@ WHERE e.team_id = %(team_id)s {event_filter} {filters} {parsed_date_from_prev_range} {parsed_date_to} {actions_query} {null_person_filter} """ +BREAKDOWN_PROP_JOIN_WITH_OTHER_SQL = """ +WHERE e.team_id = %(team_id)s {event_filter} {filters} {parsed_date_from} {parsed_date_to} {null_person_filter} + {actions_query} +""" + BREAKDOWN_PROP_JOIN_SQL = """ WHERE e.team_id = %(team_id)s {event_filter} {filters} {parsed_date_from} {parsed_date_to} {null_person_filter} + AND {breakdown_value_expr} in (%(values)s) {actions_query} """ diff --git a/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr b/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr index 35b2ecc156fe6..de25715bad65c 100644 --- a/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr +++ b/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr @@ -133,29 +133,27 @@ # name: TestBreakdowns.test_breakdown_by_event_property_with_entity_session_filter ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '') AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" - WHERE team_id = 2 - AND event = 'watched movie' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - AND (sessions.session_duration > 30.0) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '') AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND (sessions.session_duration > 30.0) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestBreakdowns.test_breakdown_by_event_property_with_entity_session_filter.1 @@ -181,13 +179,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['', 'https://example.com/', 'https://example.com'] as breakdown_value) ARRAY + (SELECT ['$$_posthog_breakdown_null_$$', 'https://example.com/', 'https://example.com'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''), (['', 'https://example.com/', 'https://example.com']), (['', 'https://example.com/', 'https://example.com']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'https://example.com/', 'https://example.com']), (['$$_posthog_breakdown_null_$$', 'https://example.com/', 'https://example.com']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT "$session_id", @@ -216,29 +214,27 @@ # name: TestBreakdowns.test_breakdown_by_session_duration_of_events ' - SELECT groupArray(value) - FROM - (SELECT sessions.session_duration AS value, - count(*) as count - FROM events e - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" - WHERE team_id = 2 - AND event = 'watched movie' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - AND (NOT has(['https://test.com'], replaceRegexpAll(JSONExtractRaw(e.properties, '$current_url'), '^"|"$', ''))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT sessions.session_duration AS value, + count(*) as count + FROM events e + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND (NOT has(['https://test.com'], replaceRegexpAll(JSONExtractRaw(e.properties, '$current_url'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestBreakdowns.test_breakdown_by_session_duration_of_events.1 @@ -436,28 +432,26 @@ # name: TestBreakdowns.test_breakdown_by_session_duration_of_unique_sessions ' - SELECT groupArray(value) - FROM - (SELECT sessions.session_duration AS value, - count(DISTINCT e."$session_id") as count - FROM events e - INNER JOIN - (SELECT "$session_id", - dateDiff('second', min(timestamp), max(timestamp)) as session_duration - FROM events - WHERE "$session_id" != '' - AND team_id = 2 - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR - GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" - WHERE team_id = 2 - AND event = 'watched movie' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT sessions.session_duration AS value, + count(DISTINCT e."$session_id") as count + FROM events e + INNER JOIN + (SELECT "$session_id", + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE "$session_id" != '' + AND team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - INTERVAL 24 HOUR + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + INTERVAL 24 HOUR + GROUP BY "$session_id") AS sessions ON sessions."$session_id" = e."$session_id" + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestBreakdowns.test_breakdown_by_session_duration_of_unique_sessions.1 @@ -660,25 +654,84 @@ ORDER BY breakdown_value ' --- -# name: TestBreakdowns.test_breakdown_numeric_hogql_to_string +# name: TestBreakdowns.test_breakdown_numeric_hogql + ' + + SELECT length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 2 + OFFSET 0 + ' +--- +# name: TestBreakdowns.test_breakdown_numeric_hogql.1 ' - SELECT groupArray(value) + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value FROM - (SELECT length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'watched movie' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-12 23:59:59', 'UTC')) - toIntervalDay(number) as day_start + FROM numbers(11) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT [1, 2] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(*) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, + transform(ifNull(length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')), 9007199254740990), ([9007199254740990, 19]), ([9007199254740990, 19]), 9007199254740991) as breakdown_value + FROM events e + WHERE e.team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')), 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value ' --- -# name: TestBreakdowns.test_breakdown_numeric_hogql_to_string.1 +# name: TestBreakdowns.test_breakdown_numeric_hogql_hide_other + ' + + SELECT length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 2 + OFFSET 0 + ' +--- +# name: TestBreakdowns.test_breakdown_numeric_hogql_hide_other.1 ' SELECT groupArray(day_start) as date, @@ -707,12 +760,260 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')), ([19, 20]), ([19, 20]), 9007199254740991) as breakdown_value + transform(ifNull(length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')), 9007199254740990), ([9007199254740990, 19]), ([9007199254740990, 19]), 9007199254740991) as breakdown_value + FROM events e + WHERE e.team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')), 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND transform(ifNull(length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')), 9007199254740990), ([9007199254740990, 19]), ([9007199254740990, 19]), 9007199254740991) in ([9007199254740990, 19]) + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value + ' +--- +# name: TestBreakdowns.test_breakdown_numeric_hogql_hide_other.2 + ' + + SELECT length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 3 + OFFSET 0 + ' +--- +# name: TestBreakdowns.test_breakdown_numeric_hogql_hide_other.3 + ' + + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value + FROM + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-12 23:59:59', 'UTC')) - toIntervalDay(number) as day_start + FROM numbers(11) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT [9007199254740990, 19, 20] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(*) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, + transform(ifNull(length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')), 9007199254740990), ([9007199254740990, 19, 20]), ([9007199254740990, 19, 20]), 9007199254740991) as breakdown_value + FROM events e + WHERE e.team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')), 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND transform(ifNull(length(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '')), 9007199254740990), ([9007199254740990, 19, 20]), ([9007199254740990, 19, 20]), 9007199254740991) in ([9007199254740990, 19, 20]) + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value + ' +--- +# name: TestBreakdowns.test_breakdown_string_hogql + ' + + SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 2 + OFFSET 0 + ' +--- +# name: TestBreakdowns.test_breakdown_string_hogql.1 + ' + + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value + FROM + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-12 23:59:59', 'UTC')) - toIntervalDay(number) as day_start + FROM numbers(11) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT ['$$_posthog_breakdown_null_$$', 'https://example.com'] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(*) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, + transform(ifNull(nullIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'https://example.com']), (['$$_posthog_breakdown_null_$$', 'https://example.com']), '$$_posthog_breakdown_other_$$') as breakdown_value + FROM events e + WHERE e.team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')), 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value + ' +--- +# name: TestBreakdowns.test_breakdown_string_hogql_hide_other + ' + + SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 2 + OFFSET 0 + ' +--- +# name: TestBreakdowns.test_breakdown_string_hogql_hide_other.1 + ' + + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value + FROM + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-12 23:59:59', 'UTC')) - toIntervalDay(number) as day_start + FROM numbers(11) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT ['$$_posthog_breakdown_null_$$', 'https://example.com'] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(*) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, + transform(ifNull(nullIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'https://example.com']), (['$$_posthog_breakdown_null_$$', 'https://example.com']), '$$_posthog_breakdown_other_$$') as breakdown_value + FROM events e + WHERE e.team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')), 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND transform(ifNull(nullIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'https://example.com']), (['$$_posthog_breakdown_null_$$', 'https://example.com']), '$$_posthog_breakdown_other_$$') in (['$$_posthog_breakdown_null_$$', 'https://example.com']) + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value + ' +--- +# name: TestBreakdowns.test_breakdown_string_hogql_hide_other.2 + ' + + SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 3 + OFFSET 0 + ' +--- +# name: TestBreakdowns.test_breakdown_string_hogql_hide_other.3 + ' + + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value + FROM + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-12 23:59:59', 'UTC')) - toIntervalDay(number) as day_start + FROM numbers(11) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT ['$$_posthog_breakdown_null_$$', 'https://example.com', 'https://example.com/'] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(*) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, + transform(ifNull(nullIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'https://example.com', 'https://example.com/']), (['$$_posthog_breakdown_null_$$', 'https://example.com', 'https://example.com/']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'watched movie' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + AND transform(ifNull(nullIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['$$_posthog_breakdown_null_$$', 'https://example.com', 'https://example.com/']), (['$$_posthog_breakdown_null_$$', 'https://example.com', 'https://example.com/']), '$$_posthog_breakdown_other_$$') in (['$$_posthog_breakdown_null_$$', 'https://example.com', 'https://example.com/']) GROUP BY day_start, breakdown_value)) GROUP BY day_start, diff --git a/posthog/queries/trends/test/__snapshots__/test_breakdowns_by_current_url.ambr b/posthog/queries/trends/test/__snapshots__/test_breakdowns_by_current_url.ambr index 811f3fe71f2aa..d02c751ab3a4d 100644 --- a/posthog/queries/trends/test/__snapshots__/test_breakdowns_by_current_url.ambr +++ b/posthog/queries/trends/test/__snapshots__/test_breakdowns_by_current_url.ambr @@ -1,21 +1,19 @@ # name: TestBreakdownsByCurrentURL.test_breakdown_by_current_url ' - SELECT groupArray(value) - FROM - (SELECT if(empty(trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))), '/', trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'watched movie' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT if(empty(trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))), '/', trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestBreakdownsByCurrentURL.test_breakdown_by_current_url.1 @@ -47,9 +45,9 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(if(empty(trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))), '/', trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))), (['https://example.com/home', 'https://example.com']), (['https://example.com/home', 'https://example.com']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(if(empty(trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))), '/', trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', ''))), ''), '$$_posthog_breakdown_null_$$'), (['https://example.com/home', 'https://example.com']), (['https://example.com/home', 'https://example.com']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'watched movie' @@ -68,21 +66,19 @@ # name: TestBreakdownsByCurrentURL.test_breakdown_by_pathname ' - SELECT groupArray(value) - FROM - (SELECT if(empty(trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))), '/', trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))) AS value, - count(*) as count - FROM events e - WHERE team_id = 2 - AND event = 'watched movie' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT if(empty(trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))), '/', trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = 'watched movie' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-12 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestBreakdownsByCurrentURL.test_breakdown_by_pathname.1 @@ -114,9 +110,9 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(if(empty(trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))), '/', trim(TRAILING '/?#' - from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))), (['/home', '/']), (['/home', '/']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(if(empty(trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))), '/', trim(TRAILING '/?#' + from replaceRegexpAll(JSONExtractRaw(properties, '$pathname'), '^"|"$', ''))), ''), '$$_posthog_breakdown_null_$$'), (['/home', '/']), (['/home', '/']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'watched movie' diff --git a/posthog/queries/trends/test/__snapshots__/test_formula.ambr b/posthog/queries/trends/test/__snapshots__/test_formula.ambr index eac1bdea575cc..a86eea7c97d85 100644 --- a/posthog/queries/trends/test/__snapshots__/test_formula.ambr +++ b/posthog/queries/trends/test/__snapshots__/test_formula.ambr @@ -21,37 +21,33 @@ # name: TestFormula.test_breakdown ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, - sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count - FROM events e - WHERE team_id = 2 - AND event = 'session start' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, + sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count + FROM events e + WHERE team_id = 2 + AND event = 'session start' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFormula.test_breakdown.1 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, - avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count - FROM events e - WHERE team_id = 2 - AND event = 'session start' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, + avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count + FROM events e + WHERE team_id = 2 + AND event = 'session start' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFormula.test_breakdown.2 @@ -87,7 +83,7 @@ day_start UNION ALL SELECT sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'session start' @@ -128,7 +124,7 @@ day_start UNION ALL SELECT avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'session start' @@ -147,37 +143,33 @@ # name: TestFormula.test_breakdown_aggregated ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, - sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count - FROM events e - WHERE team_id = 2 - AND event = 'session start' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, + sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count + FROM events e + WHERE team_id = 2 + AND event = 'session start' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFormula.test_breakdown_aggregated.1 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, - avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count - FROM events e - WHERE team_id = 2 - AND event = 'session start' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, + avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count + FROM events e + WHERE team_id = 2 + AND event = 'session start' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFormula.test_breakdown_aggregated.2 @@ -187,7 +179,7 @@ arrayFilter(x -> notEmpty(x), [replaceRegexpAll(sub_A.breakdown_value, '^"|"$', ''), replaceRegexpAll(sub_B.breakdown_value, '^"|"$', '')])[1] FROM (SELECT sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) AS total, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'session start' @@ -197,7 +189,7 @@ ORDER BY breakdown_value) as sub_A FULL OUTER JOIN (SELECT avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) AS total, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') AS breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') AS breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'session start' @@ -356,65 +348,61 @@ # name: TestFormula.test_breakdown_hogql ' - SELECT groupArray(value) - FROM - (SELECT concat(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), ' : ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'location'), ''), 'null'), '^"|"$', '')), '')) AS value, - sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT concat(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), ' : ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'location'), ''), 'null'), '^"|"$', '')), '')) AS value, + sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = 'session start' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person + WHERE team_id = 2 + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = 'session start' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFormula.test_breakdown_hogql.1 ' - SELECT groupArray(value) - FROM - (SELECT concat(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), ' : ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'location'), ''), 'null'), '^"|"$', '')), '')) AS value, - avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - INNER JOIN - (SELECT id, - argMax(properties, version) as person_props - FROM person - WHERE team_id = 2 - GROUP BY id - HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + SELECT concat(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), ' : ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'location'), ''), 'null'), '^"|"$', '')), '')) AS value, + avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 WHERE team_id = 2 - AND event = 'session start' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + INNER JOIN + (SELECT id, + argMax(properties, version) as person_props + FROM person + WHERE team_id = 2 + GROUP BY id + HAVING max(is_deleted) = 0 SETTINGS optimize_aggregation_in_order = 1) person ON pdi.person_id = person.id + WHERE team_id = 2 + AND event = 'session start' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFormula.test_breakdown_hogql.2 @@ -450,7 +438,7 @@ day_start UNION ALL SELECT sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(concat(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), ' : ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'location'), ''), 'null'), '^"|"$', '')), '')), (['some_val : London', 'some_val : Paris']), (['some_val : London', 'some_val : Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(concat(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), ' : ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'location'), ''), 'null'), '^"|"$', '')), '')), ''), '$$_posthog_breakdown_null_$$'), (['some_val : London', 'some_val : Paris']), (['some_val : London', 'some_val : Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -505,7 +493,7 @@ day_start UNION ALL SELECT avg(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(concat(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), ' : ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'location'), ''), 'null'), '^"|"$', '')), '')), (['some_val : London', 'some_val : Paris']), (['some_val : London', 'some_val : Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(concat(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), ' : ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'location'), ''), 'null'), '^"|"$', '')), '')), ''), '$$_posthog_breakdown_null_$$'), (['some_val : London', 'some_val : Paris']), (['some_val : London', 'some_val : Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e INNER JOIN (SELECT distinct_id, @@ -538,37 +526,33 @@ # name: TestFormula.test_breakdown_with_different_breakdown_values_per_series ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, - sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count - FROM events e - WHERE team_id = 2 - AND event = 'session start' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, + sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count + FROM events e + WHERE team_id = 2 + AND event = 'session start' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFormula.test_breakdown_with_different_breakdown_values_per_series.1 ' - SELECT groupArray(value) - FROM - (SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, - sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count - FROM events e - WHERE team_id = 2 - AND event = 'session end' - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 25 - OFFSET 0) + SELECT replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', '') AS value, + sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as count + FROM events e + WHERE team_id = 2 + AND event = 'session end' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2019-12-28 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-04 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0 ' --- # name: TestFormula.test_breakdown_with_different_breakdown_values_per_series.2 @@ -604,7 +588,7 @@ day_start UNION ALL SELECT sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['London', 'Paris']), (['London', 'Paris']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'session start' @@ -639,13 +623,13 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT ['London', 'Belo Horizonte', ''] as breakdown_value) ARRAY + (SELECT ['London', 'Belo Horizonte', '$$_posthog_breakdown_null_$$'] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start UNION ALL SELECT sum(toFloat64OrNull(replaceRegexpAll(JSONExtractRaw(properties, 'session duration'), '^"|"$', ''))) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - transform(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), (['London', 'Belo Horizonte', '']), (['London', 'Belo Horizonte', '']), '$$_posthog_breakdown_other_$$') as breakdown_value + transform(ifNull(nullIf(replaceRegexpAll(JSONExtractRaw(properties, 'location'), '^"|"$', ''), ''), '$$_posthog_breakdown_null_$$'), (['London', 'Belo Horizonte', '$$_posthog_breakdown_null_$$']), (['London', 'Belo Horizonte', '$$_posthog_breakdown_null_$$']), '$$_posthog_breakdown_other_$$') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = 'session end' diff --git a/posthog/queries/trends/test/test_breakdowns.py b/posthog/queries/trends/test/test_breakdowns.py index b9856947f4dc3..313ef4f37b140 100644 --- a/posthog/queries/trends/test/test_breakdowns.py +++ b/posthog/queries/trends/test/test_breakdowns.py @@ -3,13 +3,14 @@ from posthog.constants import TRENDS_TABLE from posthog.models import Filter -from posthog.queries.trends.breakdown import BREAKDOWN_OTHER_NUMERIC_LABEL -from posthog.queries.trends.trends import Trends -from posthog.test.base import ( - APIBaseTest, - ClickhouseTestMixin, - snapshot_clickhouse_queries, +from posthog.queries.trends.breakdown import ( + BREAKDOWN_OTHER_NUMERIC_LABEL, + BREAKDOWN_OTHER_STRING_LABEL, + BREAKDOWN_NULL_STRING_LABEL, + BREAKDOWN_NULL_NUMERIC_LABEL, ) +from posthog.queries.trends.trends import Trends +from posthog.test.base import APIBaseTest, ClickhouseTestMixin, snapshot_clickhouse_queries from posthog.test.test_journeys import journeys_for @@ -457,7 +458,7 @@ def test_breakdown_by_event_property_with_entity_session_filter(self): self.assertEqual( [(item["breakdown_value"], item["count"], item["data"]) for item in response], [ - ("", 6.0, [1.0, 0.0, 1.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + (BREAKDOWN_NULL_STRING_LABEL, 6.0, [1.0, 0.0, 1.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), ( "https://example.com", 1.0, @@ -493,18 +494,105 @@ def test_breakdown_histogram_by_missing_property_regression(self): ) @snapshot_clickhouse_queries - def test_breakdown_numeric_hogql_to_string(self): + def test_breakdown_numeric_hogql(self): + response = self._run( + { + "breakdown": "length(properties.$current_url)", + "breakdown_type": "hogql", + "breakdown_limit": 2, + }, + ) + self.assertEqual( + [(item["breakdown_value"], item["count"], item["data"]) for item in response], + [ + (BREAKDOWN_NULL_NUMERIC_LABEL, 6.0, [1.0, 0.0, 1.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + (19, 2.0, [2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + (BREAKDOWN_OTHER_NUMERIC_LABEL, 1.0, [1.0]), + ], + ) + + @snapshot_clickhouse_queries + def test_breakdown_numeric_hogql_hide_other(self): response = self._run( { "breakdown": "length(properties.$current_url)", "breakdown_type": "hogql", + "breakdown_hide_other_aggregation": True, + "breakdown_limit": 2, }, ) self.assertEqual( [(item["breakdown_value"], item["count"], item["data"]) for item in response], [ - (BREAKDOWN_OTHER_NUMERIC_LABEL, 6.0, [1.0, 1.0, 4.0]), + (BREAKDOWN_NULL_NUMERIC_LABEL, 6.0, [1.0, 0.0, 1.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + (19, 2.0, [2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + ], + ) + response = self._run( + { + "breakdown": "length(properties.$current_url)", + "breakdown_type": "hogql", + "breakdown_hide_other_aggregation": True, + "breakdown_limit": 3, + }, + ) + self.assertEqual( + [(item["breakdown_value"], item["count"], item["data"]) for item in response], + [ + (BREAKDOWN_NULL_NUMERIC_LABEL, 6.0, [1.0, 0.0, 1.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), (19, 2.0, [2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), (20, 1.0, [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), ], ) + + @snapshot_clickhouse_queries + def test_breakdown_string_hogql(self): + response = self._run( + { + "breakdown": "properties.$current_url", + "breakdown_type": "hogql", + "breakdown_limit": 2, + }, + ) + self.assertEqual( + [(item["breakdown_value"], item["count"], item["data"]) for item in response], + [ + (BREAKDOWN_NULL_STRING_LABEL, 6.0, [1.0, 0.0, 1.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + ("https://example.com", 2.0, [2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + (BREAKDOWN_OTHER_STRING_LABEL, 1.0, [1.0]), + ], + ) + + @snapshot_clickhouse_queries + def test_breakdown_string_hogql_hide_other(self): + response = self._run( + { + "breakdown": "properties.$current_url", + "breakdown_type": "hogql", + "breakdown_hide_other_aggregation": True, + "breakdown_limit": 2, + }, + ) + self.assertEqual( + [(item["breakdown_value"], item["count"], item["data"]) for item in response], + [ + (BREAKDOWN_NULL_STRING_LABEL, 6.0, [1.0, 0.0, 1.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + ("https://example.com", 2.0, [2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + ], + ) + response = self._run( + { + "breakdown": "properties.$current_url", + "breakdown_type": "hogql", + "breakdown_hide_other_aggregation": True, + "breakdown_limit": 3, + }, + ) + self.assertEqual( + [(item["breakdown_value"], item["count"], item["data"]) for item in response], + [ + (BREAKDOWN_NULL_STRING_LABEL, 6.0, [1.0, 0.0, 1.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + ("https://example.com", 2.0, [2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + ("https://example.com/", 1.0, [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + ], + ) diff --git a/posthog/queries/trends/test/test_formula.py b/posthog/queries/trends/test/test_formula.py index af3119826fca1..970e26692afb9 100644 --- a/posthog/queries/trends/test/test_formula.py +++ b/posthog/queries/trends/test/test_formula.py @@ -449,20 +449,21 @@ def test_breakdown_with_different_breakdown_values_per_series(self): } ) - self.assertEqual(response[0]["data"], [0.0, 0.0, 0.0, 0.0, 0.0, 400.0, 1400.0, 0.0]) self.assertEqual(response[0]["label"], "London") - self.assertEqual(response[1]["data"], [0.0, 0.0, 0.0, 0.0, 0.0, 500.0, 0.0, 0.0]) - self.assertEqual(response[1]["label"], "Paris") + self.assertEqual(response[0]["data"], [0.0, 0.0, 0.0, 0.0, 0.0, 400.0, 1400.0, 0.0]) - # regression test for empty string values - self.assertEqual(response[2]["data"], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 400.0, 0.0]) - self.assertEqual(response[2]["label"], "") + self.assertEqual(response[1]["label"], "Paris") + self.assertEqual(response[1]["data"], [0.0, 0.0, 0.0, 0.0, 0.0, 500.0, 0.0, 0.0]) # Regression test to ensure we actually get data for "Belo Horizonte" below # We previously had a bug where if series B,C,D, etc. had a value not present # in series A, we'd just default to an empty string - self.assertEqual(response[3]["data"], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 500.0, 0.0]) - self.assertEqual(response[3]["label"], "Belo Horizonte") + self.assertEqual(response[2]["label"], "Belo Horizonte") + self.assertEqual(response[2]["data"], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 500.0, 0.0]) + + # empty string values are considered "None" + self.assertEqual(response[3]["label"], "$$_posthog_breakdown_null_$$") + self.assertEqual(response[3]["data"], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 400.0, 0.0]) def test_breakdown_counts_of_different_events_one_without_events(self): with freeze_time("2020-01-04T13:01:01Z"): diff --git a/posthog/schema.py b/posthog/schema.py index 2fbc4cde6c457..4bdd25e88378d 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -756,6 +756,7 @@ class BreakdownFilter(BaseModel): ) breakdown: Optional[Union[str, float, List[Union[str, float]]]] = None breakdown_group_type_index: Optional[float] = None + breakdown_hide_other_aggregation: Optional[bool] = None breakdown_histogram_bin_count: Optional[float] = None breakdown_limit: Optional[float] = None breakdown_normalize_url: Optional[bool] = None @@ -809,6 +810,8 @@ class EventsQueryResponse(BaseModel): columns: List hasMore: Optional[bool] = None hogql: str + limit: int + offset: int results: List[List] timings: Optional[List[QueryTiming]] = None types: List[str] @@ -947,6 +950,9 @@ class PersonsQueryResponse(BaseModel): columns: List hasMore: Optional[bool] = None hogql: str + limit: int + missing_actors_count: Optional[int] = None + offset: int results: List[List] timings: Optional[List[QueryTiming]] = None types: List[str] @@ -1093,6 +1099,7 @@ class WebStatsTableQuery(BaseModel): ) breakdownBy: WebStatsBreakdown dateRange: Optional[DateRange] = None + includeScrollDepth: Optional[bool] = None kind: Literal["WebStatsTableQuery"] = "WebStatsTableQuery" properties: List[Union[EventPropertyFilter, PersonPropertyFilter]] response: Optional[WebStatsTableQueryResponse] = None @@ -1541,7 +1548,7 @@ class RetentionQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) - aggregation_group_type_index: Optional[float] = Field(default=None, description="Groups aggregation") + aggregation_group_type_index: Optional[int] = Field(default=None, description="Groups aggregation") dateRange: Optional[DateRange] = Field(default=None, description="Date range for the query") filterTestAccounts: Optional[bool] = Field( default=None, description="Exclude internal and test users by applying the respective filters" @@ -1613,7 +1620,7 @@ class TrendsQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) - aggregation_group_type_index: Optional[float] = Field(default=None, description="Groups aggregation") + aggregation_group_type_index: Optional[int] = Field(default=None, description="Groups aggregation") breakdown: Optional[BreakdownFilter] = Field(default=None, description="Breakdown of the events and actions") dateRange: Optional[DateRange] = Field(default=None, description="Date range for the query") filterTestAccounts: Optional[bool] = Field( @@ -1656,6 +1663,7 @@ class FilterType(BaseModel): aggregation_group_type_index: Optional[float] = None breakdown: Optional[Union[str, float, List[Union[str, float]]]] = None breakdown_group_type_index: Optional[float] = None + breakdown_hide_other_aggregation: Optional[bool] = None breakdown_normalize_url: Optional[bool] = None breakdown_type: Optional[BreakdownType] = None breakdowns: Optional[List[Breakdown]] = None @@ -1700,7 +1708,7 @@ class FunnelsQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) - aggregation_group_type_index: Optional[float] = Field(default=None, description="Groups aggregation") + aggregation_group_type_index: Optional[int] = Field(default=None, description="Groups aggregation") breakdown: Optional[BreakdownFilter] = Field(default=None, description="Breakdown of the events and actions") dateRange: Optional[DateRange] = Field(default=None, description="Date range for the query") filterTestAccounts: Optional[bool] = Field( @@ -1740,7 +1748,7 @@ class InsightsQueryBase(BaseModel): model_config = ConfigDict( extra="forbid", ) - aggregation_group_type_index: Optional[float] = Field(default=None, description="Groups aggregation") + aggregation_group_type_index: Optional[int] = Field(default=None, description="Groups aggregation") dateRange: Optional[DateRange] = Field(default=None, description="Date range for the query") filterTestAccounts: Optional[bool] = Field( default=None, description="Exclude internal and test users by applying the respective filters" @@ -1818,7 +1826,7 @@ class PathsQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) - aggregation_group_type_index: Optional[float] = Field(default=None, description="Groups aggregation") + aggregation_group_type_index: Optional[int] = Field(default=None, description="Groups aggregation") dateRange: Optional[DateRange] = Field(default=None, description="Date range for the query") filterTestAccounts: Optional[bool] = Field( default=None, description="Exclude internal and test users by applying the respective filters" @@ -1874,6 +1882,9 @@ class InsightPersonsQuery(BaseModel): extra="forbid", ) day: Optional[str] = None + interval: Optional[int] = Field( + default=None, description="An interval selected out of available intervals in source query" + ) kind: Literal["InsightPersonsQuery"] = "InsightPersonsQuery" response: Optional[PersonsQueryResponse] = None source: Union[TrendsQuery, FunnelsQuery, RetentionQuery, PathsQuery, StickinessQuery, LifecycleQuery] @@ -1901,8 +1912,8 @@ class PersonsQuery(BaseModel): ] ] = None kind: Literal["PersonsQuery"] = "PersonsQuery" - limit: Optional[float] = None - offset: Optional[float] = None + limit: Optional[int] = None + offset: Optional[int] = None orderBy: Optional[List[str]] = None properties: Optional[ List[ diff --git a/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py index c39eee18bf79c..f4b6135805845 100644 --- a/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py +++ b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py @@ -85,8 +85,12 @@ def __init__( SELECT distinct log_source_id as session_id FROM log_entries PREWHERE team_id = %(team_id)s + -- regardless of what other filters are applied + -- limit by storage TTL AND timestamp >= %(clamped_to_storage_ttl)s + -- make sure we don't get the occasional unexpected future event AND timestamp <= now() + -- and then any time filter for the events query {events_timestamp_clause} WHERE 1=1 {console_log_clause} @@ -274,10 +278,12 @@ def ttl_days(self): -- regardless of what other filters are applied -- limit by storage TTL AND e.timestamp >= %(clamped_to_storage_ttl)s + -- make sure we don't get the occasional unexpected future event AND e.timestamp <= now() + -- and then any time filter for the events query + {events_timestamp_clause} WHERE notEmpty(`$session_id`) - {events_timestamp_clause} {event_filter_where_conditions} {prop_filter_clause} {provided_session_ids_clause} diff --git a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr index 19e9877668818..745b5a4e444c7 100644 --- a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr +++ b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr @@ -29,9 +29,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2022-12-14 00:00:00' AND e.timestamp <= now() + AND timestamp >= '2022-12-27 12:00:00' + AND timestamp <= '2023-01-04 12:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2022-12-27 12:00:00' - AND timestamp <= '2023-01-04 12:00:00' AND (((event = 'custom-event' AND (has(['Firefox'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')) AND has(['test_action_filter-session-one'], "$session_id") @@ -77,9 +77,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2022-12-14 00:00:00' AND e.timestamp <= now() + AND timestamp >= '2022-12-27 12:00:00' + AND timestamp <= '2023-01-04 12:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2022-12-27 12:00:00' - AND timestamp <= '2023-01-04 12:00:00' AND (((event = 'custom-event' AND (has(['test_action_filter-session-one'], "$session_id") AND has(['test_action_filter-window-id'], "$window_id"))))) @@ -124,9 +124,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2022-12-14 00:00:00' AND e.timestamp <= now() + AND timestamp >= '2022-12-27 12:00:00' + AND timestamp <= '2023-01-04 12:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2022-12-27 12:00:00' - AND timestamp <= '2023-01-04 12:00:00' AND (((event = 'custom-event' AND (has(['test_action_filter-session-one'], "$session_id") AND has(['test_action_filter-window-id'], "$window_id")))) @@ -172,9 +172,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2022-12-14 00:00:00' AND e.timestamp <= now() + AND timestamp >= '2022-12-27 12:00:00' + AND timestamp <= '2023-01-04 12:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2022-12-27 12:00:00' - AND timestamp <= '2023-01-04 12:00:00' AND (((event = 'custom-event' AND (has(['test_action_filter-session-one'], "$session_id") AND has(['test_action_filter-window-id'], "$window_id")))) @@ -230,9 +230,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-21 12:00:00' + AND timestamp <= '2021-01-05 11:59:59' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-21 12:00:00' - AND timestamp <= '2021-01-05 11:59:59' AND ((event = '$pageview') OR ((event = 'custom-event'))) AND e.distinct_id in @@ -286,9 +286,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (1 = 1) GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) @@ -329,9 +329,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (1 = 1 AND (has(['Chrome'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')))) GROUP BY `$session_id` @@ -373,9 +373,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (1 = 1 AND (has(['Firefox'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')))) GROUP BY `$session_id` @@ -417,9 +417,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (1 = 1) GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) @@ -460,9 +460,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (1 = 1 AND (has(['Chrome'], "mat_$browser"))) GROUP BY `$session_id` @@ -504,9 +504,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (1 = 1 AND (has(['Firefox'], "mat_$browser"))) GROUP BY `$session_id` @@ -1034,9 +1034,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -1079,9 +1079,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$autocapture') GROUP BY `$session_id` HAVING 1=1 @@ -1124,9 +1124,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -1199,9 +1199,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -1245,9 +1245,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -1298,9 +1298,9 @@ GROUP BY group_key) groups_1 ON "$group_1" == groups_1.group_key PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview' AND (has(['org one'], replaceRegexpAll(JSONExtractRaw(group_properties_1, 'name'), '^"|"$', '')))) GROUP BY `$session_id` @@ -1344,9 +1344,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -1407,9 +1407,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') AND e.distinct_id in (select distinct_id @@ -1487,9 +1487,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Chrome'), 0)) AND e.distinct_id in (select distinct_id @@ -1549,9 +1549,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -1611,9 +1611,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') AND e.distinct_id in (select distinct_id @@ -1689,9 +1689,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (ifNull(equals(nullIf(nullIf(events.`mat_$browser`, ''), 'null'), 'Chrome'), 0)) AND e.distinct_id in (select distinct_id @@ -1782,9 +1782,9 @@ HAVING argMax(is_deleted, version) = 0) as pdi on pdi.distinct_id = e.distinct_id PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'email'), ''), 'null'), '^"|"$', ''), 'bla'), 0))) GROUP BY `$session_id` @@ -1860,9 +1860,9 @@ HAVING argMax(is_deleted, version) = 0) as pdi on pdi.distinct_id = e.distinct_id PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'email'), ''), 'null'), '^"|"$', ''), 'something else'), 0))) GROUP BY `$session_id` @@ -1906,9 +1906,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Chrome'), 0))) GROUP BY `$session_id` @@ -1952,9 +1952,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0))) GROUP BY `$session_id` @@ -1998,9 +1998,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (ifNull(equals(nullIf(nullIf(events.`mat_$browser`, ''), 'null'), 'Chrome'), 0))) GROUP BY `$session_id` @@ -2044,9 +2044,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (ifNull(equals(nullIf(nullIf(events.`mat_$browser`, ''), 'null'), 'Firefox'), 0))) GROUP BY `$session_id` @@ -2090,9 +2090,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -2135,9 +2135,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$autocapture') GROUP BY `$session_id` HAVING 1=1 @@ -2231,9 +2231,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (has(['Chrome'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')))) GROUP BY `$session_id` @@ -2277,9 +2277,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (has(['Firefox'], replaceRegexpAll(JSONExtractRaw(properties, '$browser'), '^"|"$', '')))) GROUP BY `$session_id` @@ -2323,9 +2323,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (has(['Chrome'], "mat_$browser"))) GROUP BY `$session_id` @@ -2369,9 +2369,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND (event = '$pageview' AND (has(['Firefox'], "mat_$browser"))) GROUP BY `$session_id` @@ -2436,9 +2436,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') AND (has(['false'], replaceRegexpAll(JSONExtractRaw(e.properties, 'is_internal_user'), '^"|"$', '')) AND ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Chrome'), 0)) @@ -2504,9 +2504,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -2570,9 +2570,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') AND (has(['false'], "mat_is_internal_user") AND ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Chrome'), 0)) @@ -2638,9 +2638,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -2683,9 +2683,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND ((event = '$pageview') OR event = '$pageleave') GROUP BY `$session_id` @@ -2906,131 +2906,6 @@ OFFSET 0 ' --- -# name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.5 - ' - - SELECT s.session_id, - any(s.team_id), - any(s.distinct_id), - min(s.min_first_timestamp) as start_time, - max(s.max_last_timestamp) as end_time, - dateDiff('SECOND', start_time, end_time) as duration, - argMinMerge(s.first_url) as first_url, - sum(s.click_count), - sum(s.keypress_count), - sum(s.mouse_activity_count), - sum(s.active_milliseconds)/1000 as active_seconds, - duration-active_seconds as inactive_seconds, - sum(s.console_log_count) as console_log_count, - sum(s.console_warn_count) as console_warn_count, - sum(s.console_error_count) as console_error_count - FROM session_replay_events s - WHERE s.team_id = 2 - AND s.min_first_timestamp >= '2020-12-31 20:00:00' - AND s.min_first_timestamp >= '2021-01-14 00:00:00' - AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in ['with-warns-session'] - GROUP BY session_id - HAVING 1=1 - AND (console_warn_count > 0 - OR console_error_count > 0) - ORDER BY start_time DESC - LIMIT 51 - OFFSET 0 - ' ---- -# name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.6 - ' - - SELECT distinct log_source_id as session_id - FROM log_entries PREWHERE team_id = 2 - AND timestamp >= '2020-12-31 20:00:00' - AND timestamp <= now() - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' - WHERE 1=1 - AND level in ['log'] - AND positionCaseInsensitive(message, 'message 5') > 0 - ' ---- -# name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.7 - ' - - SELECT s.session_id, - any(s.team_id), - any(s.distinct_id), - min(s.min_first_timestamp) as start_time, - max(s.max_last_timestamp) as end_time, - dateDiff('SECOND', start_time, end_time) as duration, - argMinMerge(s.first_url) as first_url, - sum(s.click_count), - sum(s.keypress_count), - sum(s.mouse_activity_count), - sum(s.active_milliseconds)/1000 as active_seconds, - duration-active_seconds as inactive_seconds, - sum(s.console_log_count) as console_log_count, - sum(s.console_warn_count) as console_warn_count, - sum(s.console_error_count) as console_error_count - FROM session_replay_events s - WHERE s.team_id = 2 - AND s.min_first_timestamp >= '2020-12-31 20:00:00' - AND s.min_first_timestamp >= '2021-01-14 00:00:00' - AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in [] - GROUP BY session_id - HAVING 1=1 - AND (console_log_count > 0) - ORDER BY start_time DESC - LIMIT 51 - OFFSET 0 - ' ---- -# name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.8 - ' - - SELECT distinct log_source_id as session_id - FROM log_entries PREWHERE team_id = 2 - AND timestamp >= '2020-12-31 20:00:00' - AND timestamp <= now() - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' - WHERE 1=1 - AND level in ['log'] - AND positionCaseInsensitive(message, 'message 5') > 0 - ' ---- -# name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.9 - ' - - SELECT s.session_id, - any(s.team_id), - any(s.distinct_id), - min(s.min_first_timestamp) as start_time, - max(s.max_last_timestamp) as end_time, - dateDiff('SECOND', start_time, end_time) as duration, - argMinMerge(s.first_url) as first_url, - sum(s.click_count), - sum(s.keypress_count), - sum(s.mouse_activity_count), - sum(s.active_milliseconds)/1000 as active_seconds, - duration-active_seconds as inactive_seconds, - sum(s.console_log_count) as console_log_count, - sum(s.console_warn_count) as console_warn_count, - sum(s.console_error_count) as console_error_count - FROM session_replay_events s - WHERE s.team_id = 2 - AND s.min_first_timestamp >= '2020-12-31 20:00:00' - AND s.min_first_timestamp >= '2021-01-14 00:00:00' - AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in [] - GROUP BY session_id - HAVING 1=1 - AND (console_log_count > 0) - ORDER BY start_time DESC - LIMIT 51 - OFFSET 0 - ' ---- # name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_with_console_errors ' @@ -3547,9 +3422,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2021-07-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-08-13 12:00:00' + AND timestamp <= '2021-08-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-08-13 12:00:00' - AND timestamp <= '2021-08-22 08:00:00' AND (event = '$pageview') AND e.distinct_id in (select distinct_id @@ -3634,9 +3509,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2021-07-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-08-13 12:00:00' + AND timestamp <= '2021-08-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-08-13 12:00:00' - AND timestamp <= '2021-08-22 08:00:00' AND (event = 'custom_event') AND e.distinct_id in (select distinct_id @@ -3700,9 +3575,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND ((event = '$pageview') OR event = 'new-event') GROUP BY `$session_id` @@ -3746,9 +3621,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-11 13:46:23' AND e.timestamp <= now() + AND timestamp >= '2020-12-24 12:00:00' + AND timestamp <= '2021-01-02 01:46:23' WHERE notEmpty(`$session_id`) - AND timestamp >= '2020-12-24 12:00:00' - AND timestamp <= '2021-01-02 01:46:23' AND ((event = '$pageview') OR event = 'new-event2') GROUP BY `$session_id` @@ -3832,9 +3707,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -3876,9 +3751,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (has(['false'], replaceRegexpAll(JSONExtractRaw(e.properties, 'is_internal_user'), '^"|"$', ''))) GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) @@ -3920,9 +3795,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -3964,9 +3839,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (has(['false'], "mat_is_internal_user")) GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) @@ -4008,9 +3883,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -4052,9 +3927,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'is_internal_user'), ''), 'null'), '^"|"$', ''), 'true'), 0)) GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) @@ -4096,9 +3971,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -4140,9 +4015,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (ifNull(equals(nullIf(nullIf(events.mat_is_internal_user, ''), 'null'), 'true'), 0)) GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) @@ -4184,9 +4059,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -4277,9 +4152,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -4369,9 +4244,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 @@ -4465,9 +4340,9 @@ FROM events e PREWHERE team_id = 2 AND e.timestamp >= '2020-12-31 20:00:00' AND e.timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' WHERE notEmpty(`$session_id`) - AND timestamp >= '2021-01-13 12:00:00' - AND timestamp <= '2021-01-22 08:00:00' AND (event = '$pageview') GROUP BY `$session_id` HAVING 1=1 diff --git a/posthog/session_recordings/session_recording_api.py b/posthog/session_recordings/session_recording_api.py index 87f4a12a2c5ed..915fbc83d68ba 100644 --- a/posthog/session_recordings/session_recording_api.py +++ b/posthog/session_recordings/session_recording_api.py @@ -1,5 +1,5 @@ import time -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import json from typing import Any, List, Type, cast, Dict, Tuple @@ -352,7 +352,7 @@ def snapshots(self, request: request.Request, **kwargs): for full_key in blob_keys: # Keys are like 1619712000-1619712060 blob_key = full_key.replace(blob_prefix.rstrip("/") + "/", "") - time_range = [datetime.fromtimestamp(int(x) / 1000) for x in blob_key.split("-")] + time_range = [datetime.fromtimestamp(int(x) / 1000, tz=timezone.utc) for x in blob_key.split("-")] sources.append( { @@ -369,7 +369,7 @@ def snapshots(self, request: request.Request, **kwargs): newest_timestamp = min(sources, key=lambda k: k["end_timestamp"])["end_timestamp"] if might_have_realtime: - might_have_realtime = oldest_timestamp + timedelta(hours=24) > datetime.utcnow() + might_have_realtime = oldest_timestamp + timedelta(hours=24) > datetime.now(timezone.utc) if might_have_realtime: sources.append( diff --git a/posthog/session_recordings/sql/session_replay_event_sql.py b/posthog/session_recordings/sql/session_replay_event_sql.py index 6c9a1e76bf252..ccec36c2b9f39 100644 --- a/posthog/session_recordings/sql/session_replay_event_sql.py +++ b/posthog/session_recordings/sql/session_replay_event_sql.py @@ -38,6 +38,8 @@ ) ENGINE = {engine} """ +# if updating these column definitions +# you'll need to update the explicit column definitions in the materialized view creation statement below SESSION_REPLAY_EVENTS_TABLE_BASE_SQL = """ CREATE TABLE IF NOT EXISTS {table_name} ON CLUSTER '{cluster}' ( @@ -107,11 +109,10 @@ engine=kafka_engine(topic=KAFKA_CLICKHOUSE_SESSION_REPLAY_EVENTS), ) - SESSION_REPLAY_EVENTS_TABLE_MV_SQL = ( lambda: """ CREATE MATERIALIZED VIEW IF NOT EXISTS session_replay_events_mv ON CLUSTER '{cluster}' -TO {database}.{target_table} +TO {database}.{target_table} {explictly_specify_columns} AS SELECT session_id, team_id, @@ -147,10 +148,25 @@ target_table="writable_session_replay_events", cluster=settings.CLICKHOUSE_CLUSTER, database=settings.CLICKHOUSE_DATABASE, + # ClickHouse is incorrectly expanding the type of the snapshot source column + # Despite it being a LowCardinality(Nullable(String)) in writable_session_replay_events + # The column expansion picks only Nullable(String) and so we can't select it + explictly_specify_columns="""( +`session_id` String, `team_id` Int64, `distinct_id` String, +`min_first_timestamp` DateTime64(6, 'UTC'), +`max_last_timestamp` DateTime64(6, 'UTC'), +`first_url` AggregateFunction(argMin, Nullable(String), DateTime64(6, 'UTC')), +`click_count` Int64, `keypress_count` Int64, +`mouse_activity_count` Int64, `active_milliseconds` Int64, +`console_log_count` Int64, `console_warn_count` Int64, +`console_error_count` Int64, `size` Int64, `message_count` Int64, +`event_count` Int64, +`snapshot_source` AggregateFunction(argMin, LowCardinality(Nullable(String)), DateTime64(6, 'UTC')), +`_timestamp` Nullable(DateTime) +)""", ) ) - # Distributed engine tables are only created if CLICKHOUSE_REPLICATED # This table is responsible for writing to sharded_session_replay_events based on a sharding key. @@ -163,7 +179,6 @@ ), ) - # This table is responsible for reading from session_replay_events on a cluster setting DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL = lambda: SESSION_REPLAY_EVENTS_TABLE_BASE_SQL.format( table_name="session_replay_events", @@ -174,7 +189,6 @@ ), ) - DROP_SESSION_REPLAY_EVENTS_TABLE_SQL = lambda: ( f"DROP TABLE IF EXISTS {SESSION_REPLAY_EVENTS_DATA_TABLE()} ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}'" ) diff --git a/posthog/tasks/__init__.py b/posthog/tasks/__init__.py index 261a4c33ef1a5..80d3661259f16 100644 --- a/posthog/tasks/__init__.py +++ b/posthog/tasks/__init__.py @@ -7,6 +7,7 @@ demo_create_data, email, exporter, + process_scheduled_changes, split_person, sync_all_organization_available_features, usage_report, @@ -20,6 +21,7 @@ "demo_create_data", "email", "exporter", + "process_scheduled_changes", "split_person", "sync_all_organization_available_features", "user_identify", diff --git a/posthog/tasks/email.py b/posthog/tasks/email.py index 6ab43b973f509..4b07bdc63271d 100644 --- a/posthog/tasks/email.py +++ b/posthog/tasks/email.py @@ -119,7 +119,7 @@ def send_email_verification(user_id: int, token: str) -> None: }, ) message.add_recipient(user.pending_email if user.pending_email is not None else user.email) - message.send() + message.send(send_async=False) posthoganalytics.capture( user.distinct_id, "verification email sent", diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index c0dc99ff436fc..83cf709e17bd1 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -120,11 +120,12 @@ def _convert_response_to_csv_data(data: Any) -> List[Any]: return csv_rows elif first_result.get("appearances") and first_result.get("person"): # RETENTION PERSONS LIKE + period = data["filters"]["period"] or "Day" csv_rows = [] for item in items: line = {"person": item["person"]["name"]} for index, data in enumerate(item["appearances"]): - line[f"Day {index}"] = data + line[f"{period} {index}"] = data csv_rows.append(line) return csv_rows diff --git a/posthog/tasks/exports/test/csv_renders/persons_modal_retention.json b/posthog/tasks/exports/test/csv_renders/persons_modal_retention.json index 11a1f07bb700b..10caeefcba95f 100644 --- a/posthog/tasks/exports/test/csv_renders/persons_modal_retention.json +++ b/posthog/tasks/exports/test/csv_renders/persons_modal_retention.json @@ -1,6 +1,6 @@ { "csv_rows": [ - "person,Day 0,Day 1,Day 2,Day 3,Day 4,Day 5,Day 6,Day 7,Day 8,Day 9,Day 10", + "person,Week 0,Week 1,Week 2,Week 3,Week 4,Week 5,Week 6,Week 7,Week 8,Week 9,Week 10", "talent1974@yahoo.com,1,1,1,1,1,1,1,1,1,1,0", "few2035@protonmail.com,1,1,1,1,1,1,1,1,1,0,0", "" @@ -47,6 +47,43 @@ } ], "next": null, - "missing_persons": 0 + "missing_persons": 0, + "filters": { + "breakdown_attribution_type": "first_touch", + "breakdown_normalize_url": false, + "breakdown_values": [0], + "date_from": "-11d", + "display": "ActionsTable", + "insight": "RETENTION", + "limit": 100, + "period": "Week", + "retention_type": "retention_first_time", + "returning_entity": { + "id": "$pageview", + "type": "events", + "order": null, + "name": "$pageview", + "custom_name": null, + "math": null, + "math_property": null, + "math_hogql": null, + "math_group_type_index": null, + "properties": {} + }, + "sampling_factor": "", + "target_entity": { + "id": "$pageview", + "type": "events", + "order": null, + "name": "$pageview", + "custom_name": null, + "math": null, + "math_property": null, + "math_hogql": null, + "math_group_type_index": null, + "properties": {} + }, + "total_intervals": 11 + } } } diff --git a/posthog/tasks/exports/test/test_csv_exporter.py b/posthog/tasks/exports/test/test_csv_exporter.py index 65fda3baa0dd4..3570d19440021 100644 --- a/posthog/tasks/exports/test/test_csv_exporter.py +++ b/posthog/tasks/exports/test/test_csv_exporter.py @@ -292,7 +292,7 @@ def test_raises_expected_error_when_json_is_none(self, patched_api_call) -> None @patch("posthog.hogql.constants.DEFAULT_RETURNED_ROWS", 5) @patch("posthog.models.exported_asset.UUIDT") def test_csv_exporter_hogql_query(self, mocked_uuidt, DEFAULT_RETURNED_ROWS=5, MAX_SELECT_RETURNED_ROWS=10) -> None: - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" for i in range(15): _create_event( event="$pageview", @@ -335,7 +335,7 @@ def test_csv_exporter_hogql_query(self, mocked_uuidt, DEFAULT_RETURNED_ROWS=5, M @patch("posthog.hogql.constants.MAX_SELECT_RETURNED_ROWS", 10) @patch("posthog.models.exported_asset.UUIDT") def test_csv_exporter_events_query(self, mocked_uuidt, MAX_SELECT_RETURNED_ROWS=10) -> None: - random_uuid = str(UUIDT()) + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" for i in range(15): _create_event( event="$pageview", diff --git a/posthog/tasks/process_scheduled_changes.py b/posthog/tasks/process_scheduled_changes.py new file mode 100644 index 0000000000000..22d09e9948d35 --- /dev/null +++ b/posthog/tasks/process_scheduled_changes.py @@ -0,0 +1,39 @@ +from posthog.models import ScheduledChange +from django.utils import timezone +from posthog.models import FeatureFlag +from django.db import transaction, OperationalError + +models = {"FeatureFlag": FeatureFlag} + + +def process_scheduled_changes() -> None: + try: + with transaction.atomic(): + scheduled_changes = ( + ScheduledChange.objects.select_for_update(nowait=True) + .filter( + executed_at__isnull=True, + scheduled_at__lte=timezone.now(), + ) + .order_by("scheduled_at")[:10000] + ) + + for scheduled_change in scheduled_changes: + try: + # Execute the change on the model instance + model = models[scheduled_change.model_name] + instance = model.objects.get(id=scheduled_change.record_id) + instance.scheduled_changes_dispatcher(scheduled_change.payload) + + # Mark scheduled change completed + scheduled_change.executed_at = timezone.now() + scheduled_change.save() + + except Exception as e: + # Store the failure reason + scheduled_change.failure_reason = str(e) + scheduled_change.executed_at = timezone.now() + scheduled_change.save() + except OperationalError: + # Failed to obtain the lock + pass diff --git a/posthog/tasks/test/__snapshots__/test_process_scheduled_changes.ambr b/posthog/tasks/test/__snapshots__/test_process_scheduled_changes.ambr new file mode 100644 index 0000000000000..87019fd274336 --- /dev/null +++ b/posthog/tasks/test/__snapshots__/test_process_scheduled_changes.ambr @@ -0,0 +1,291 @@ +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes + ' + SELECT "posthog_featureflag"."id", + "posthog_featureflag"."key", + "posthog_featureflag"."name", + "posthog_featureflag"."filters", + "posthog_featureflag"."rollout_percentage", + "posthog_featureflag"."team_id", + "posthog_featureflag"."created_by_id", + "posthog_featureflag"."created_at", + "posthog_featureflag"."deleted", + "posthog_featureflag"."active", + "posthog_featureflag"."rollback_conditions", + "posthog_featureflag"."performed_rollback", + "posthog_featureflag"."ensure_experience_continuity", + "posthog_featureflag"."usage_dashboard_id", + "posthog_featureflag"."has_enriched_analytics" + FROM "posthog_featureflag" + WHERE ("posthog_featureflag"."active" + AND NOT "posthog_featureflag"."deleted" + AND "posthog_featureflag"."team_id" = 2) + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.1 + ' + SELECT "posthog_scheduledchange"."id", + "posthog_scheduledchange"."record_id", + "posthog_scheduledchange"."model_name", + "posthog_scheduledchange"."payload", + "posthog_scheduledchange"."scheduled_at", + "posthog_scheduledchange"."executed_at", + "posthog_scheduledchange"."failure_reason", + "posthog_scheduledchange"."team_id", + "posthog_scheduledchange"."created_at", + "posthog_scheduledchange"."created_by_id", + "posthog_scheduledchange"."updated_at" + FROM "posthog_scheduledchange" + WHERE ("posthog_scheduledchange"."executed_at" IS NULL + AND "posthog_scheduledchange"."scheduled_at" <= '2023-12-21T09:00:00+00:00'::timestamptz) + ORDER BY "posthog_scheduledchange"."scheduled_at" ASC + LIMIT 10000 + FOR + UPDATE NOWAIT + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.10 + ' + SELECT "posthog_scheduledchange"."id", + "posthog_scheduledchange"."record_id", + "posthog_scheduledchange"."model_name", + "posthog_scheduledchange"."payload", + "posthog_scheduledchange"."scheduled_at", + "posthog_scheduledchange"."executed_at", + "posthog_scheduledchange"."failure_reason", + "posthog_scheduledchange"."team_id", + "posthog_scheduledchange"."created_at", + "posthog_scheduledchange"."created_by_id", + "posthog_scheduledchange"."updated_at" + FROM "posthog_scheduledchange" + WHERE "posthog_scheduledchange"."id" = 2 + LIMIT 21 + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.11 + ' + SELECT "posthog_scheduledchange"."id", + "posthog_scheduledchange"."record_id", + "posthog_scheduledchange"."model_name", + "posthog_scheduledchange"."payload", + "posthog_scheduledchange"."scheduled_at", + "posthog_scheduledchange"."executed_at", + "posthog_scheduledchange"."failure_reason", + "posthog_scheduledchange"."team_id", + "posthog_scheduledchange"."created_at", + "posthog_scheduledchange"."created_by_id", + "posthog_scheduledchange"."updated_at" + FROM "posthog_scheduledchange" + WHERE "posthog_scheduledchange"."id" = 2 + LIMIT 21 + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.12 + ' + SELECT "posthog_featureflag"."id", + "posthog_featureflag"."key", + "posthog_featureflag"."name", + "posthog_featureflag"."filters", + "posthog_featureflag"."rollout_percentage", + "posthog_featureflag"."team_id", + "posthog_featureflag"."created_by_id", + "posthog_featureflag"."created_at", + "posthog_featureflag"."deleted", + "posthog_featureflag"."active", + "posthog_featureflag"."rollback_conditions", + "posthog_featureflag"."performed_rollback", + "posthog_featureflag"."ensure_experience_continuity", + "posthog_featureflag"."usage_dashboard_id", + "posthog_featureflag"."has_enriched_analytics" + FROM "posthog_featureflag" + WHERE "posthog_featureflag"."key" = 'flag-1' + LIMIT 21 + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.2 + ' + SELECT "posthog_featureflag"."id", + "posthog_featureflag"."key", + "posthog_featureflag"."name", + "posthog_featureflag"."filters", + "posthog_featureflag"."rollout_percentage", + "posthog_featureflag"."team_id", + "posthog_featureflag"."created_by_id", + "posthog_featureflag"."created_at", + "posthog_featureflag"."deleted", + "posthog_featureflag"."active", + "posthog_featureflag"."rollback_conditions", + "posthog_featureflag"."performed_rollback", + "posthog_featureflag"."ensure_experience_continuity", + "posthog_featureflag"."usage_dashboard_id", + "posthog_featureflag"."has_enriched_analytics" + FROM "posthog_featureflag" + WHERE "posthog_featureflag"."id" = 2 + LIMIT 21 + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.3 + ' + SELECT "posthog_user"."id", + "posthog_user"."password", + "posthog_user"."last_login", + "posthog_user"."first_name", + "posthog_user"."last_name", + "posthog_user"."is_staff", + "posthog_user"."is_active", + "posthog_user"."date_joined", + "posthog_user"."uuid", + "posthog_user"."current_organization_id", + "posthog_user"."current_team_id", + "posthog_user"."email", + "posthog_user"."pending_email", + "posthog_user"."temporary_token", + "posthog_user"."distinct_id", + "posthog_user"."is_email_verified", + "posthog_user"."requested_password_reset_at", + "posthog_user"."has_seen_product_intro_for", + "posthog_user"."email_opt_in", + "posthog_user"."theme_mode", + "posthog_user"."partial_notification_settings", + "posthog_user"."anonymize_data", + "posthog_user"."toolbar_mode", + "posthog_user"."events_column_config" + FROM "posthog_user" + WHERE "posthog_user"."id" = 2 + LIMIT 21 + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.4 + ' + SELECT "posthog_featureflag"."id", + "posthog_featureflag"."key", + "posthog_featureflag"."name", + "posthog_featureflag"."filters", + "posthog_featureflag"."rollout_percentage", + "posthog_featureflag"."team_id", + "posthog_featureflag"."created_by_id", + "posthog_featureflag"."created_at", + "posthog_featureflag"."deleted", + "posthog_featureflag"."active", + "posthog_featureflag"."rollback_conditions", + "posthog_featureflag"."performed_rollback", + "posthog_featureflag"."ensure_experience_continuity", + "posthog_featureflag"."usage_dashboard_id", + "posthog_featureflag"."has_enriched_analytics" + FROM "posthog_featureflag" + WHERE ("posthog_featureflag"."active" + AND NOT "posthog_featureflag"."deleted" + AND "posthog_featureflag"."team_id" = 2) + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.5 + ' + SELECT "posthog_featureflag"."id", + "posthog_featureflag"."key", + "posthog_featureflag"."name", + "posthog_featureflag"."filters", + "posthog_featureflag"."rollout_percentage", + "posthog_featureflag"."team_id", + "posthog_featureflag"."created_by_id", + "posthog_featureflag"."created_at", + "posthog_featureflag"."deleted", + "posthog_featureflag"."active", + "posthog_featureflag"."rollback_conditions", + "posthog_featureflag"."performed_rollback", + "posthog_featureflag"."ensure_experience_continuity", + "posthog_featureflag"."usage_dashboard_id", + "posthog_featureflag"."has_enriched_analytics" + FROM "posthog_featureflag" + WHERE "posthog_featureflag"."id" = 2 + LIMIT 21 + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.6 + ' + SELECT "posthog_user"."id", + "posthog_user"."password", + "posthog_user"."last_login", + "posthog_user"."first_name", + "posthog_user"."last_name", + "posthog_user"."is_staff", + "posthog_user"."is_active", + "posthog_user"."date_joined", + "posthog_user"."uuid", + "posthog_user"."current_organization_id", + "posthog_user"."current_team_id", + "posthog_user"."email", + "posthog_user"."pending_email", + "posthog_user"."temporary_token", + "posthog_user"."distinct_id", + "posthog_user"."is_email_verified", + "posthog_user"."requested_password_reset_at", + "posthog_user"."has_seen_product_intro_for", + "posthog_user"."email_opt_in", + "posthog_user"."theme_mode", + "posthog_user"."partial_notification_settings", + "posthog_user"."anonymize_data", + "posthog_user"."toolbar_mode", + "posthog_user"."events_column_config" + FROM "posthog_user" + WHERE "posthog_user"."id" = 2 + LIMIT 21 + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.7 + ' + SELECT "posthog_featureflag"."id", + "posthog_featureflag"."key", + "posthog_featureflag"."name", + "posthog_featureflag"."filters", + "posthog_featureflag"."rollout_percentage", + "posthog_featureflag"."team_id", + "posthog_featureflag"."created_by_id", + "posthog_featureflag"."created_at", + "posthog_featureflag"."deleted", + "posthog_featureflag"."active", + "posthog_featureflag"."rollback_conditions", + "posthog_featureflag"."performed_rollback", + "posthog_featureflag"."ensure_experience_continuity", + "posthog_featureflag"."usage_dashboard_id", + "posthog_featureflag"."has_enriched_analytics" + FROM "posthog_featureflag" + WHERE ("posthog_featureflag"."active" + AND NOT "posthog_featureflag"."deleted" + AND "posthog_featureflag"."team_id" = 2) + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.8 + ' + SELECT "posthog_scheduledchange"."id", + "posthog_scheduledchange"."record_id", + "posthog_scheduledchange"."model_name", + "posthog_scheduledchange"."payload", + "posthog_scheduledchange"."scheduled_at", + "posthog_scheduledchange"."executed_at", + "posthog_scheduledchange"."failure_reason", + "posthog_scheduledchange"."team_id", + "posthog_scheduledchange"."created_at", + "posthog_scheduledchange"."created_by_id", + "posthog_scheduledchange"."updated_at" + FROM "posthog_scheduledchange" + WHERE "posthog_scheduledchange"."id" = 2 + LIMIT 21 + ' +--- +# name: TestProcessScheduledChanges.test_schedule_feature_flag_multiple_changes.9 + ' + SELECT "posthog_scheduledchange"."id", + "posthog_scheduledchange"."record_id", + "posthog_scheduledchange"."model_name", + "posthog_scheduledchange"."payload", + "posthog_scheduledchange"."scheduled_at", + "posthog_scheduledchange"."executed_at", + "posthog_scheduledchange"."failure_reason", + "posthog_scheduledchange"."team_id", + "posthog_scheduledchange"."created_at", + "posthog_scheduledchange"."created_by_id", + "posthog_scheduledchange"."updated_at" + FROM "posthog_scheduledchange" + WHERE "posthog_scheduledchange"."id" = 2 + LIMIT 21 + ' +--- diff --git a/posthog/tasks/test/test_process_scheduled_changes.py b/posthog/tasks/test/test_process_scheduled_changes.py new file mode 100644 index 0000000000000..866f3847c5d34 --- /dev/null +++ b/posthog/tasks/test/test_process_scheduled_changes.py @@ -0,0 +1,179 @@ +from datetime import datetime, timedelta, timezone +from posthog.models import ScheduledChange, FeatureFlag +from posthog.test.base import APIBaseTest, QueryMatchingTest, snapshot_postgres_queries +from posthog.tasks.process_scheduled_changes import process_scheduled_changes +from freezegun import freeze_time + + +class TestProcessScheduledChanges(APIBaseTest, QueryMatchingTest): + def test_schedule_feature_flag_set_active(self) -> None: + feature_flag = FeatureFlag.objects.create( + name="Flag 1", + key="flag-1", + active=False, + filters={"groups": []}, + team=self.team, + created_by=self.user, + ) + + ScheduledChange.objects.create( + team=self.team, + record_id=feature_flag.id, + model_name="FeatureFlag", + payload={"operation": "update_status", "value": True}, + scheduled_at=(datetime.now(timezone.utc) - timedelta(seconds=30)).isoformat(), + ) + + process_scheduled_changes() + + updated_flag = FeatureFlag.objects.get(key="flag-1") + self.assertEqual(updated_flag.active, True) + + def test_schedule_feature_flag_add_release_condition(self) -> None: + feature_flag = FeatureFlag.objects.create( + name="Flag 1", + key="flag-1", + active=False, + filters={"groups": []}, + team=self.team, + created_by=self.user, + ) + + new_release_condition = { + "variant": None, + "properties": [{"key": "$browser", "type": "person", "value": ["Chrome"], "operator": "exact"}], + "rollout_percentage": 30, + } + + payload = { + "operation": "add_release_condition", + "value": {"groups": [new_release_condition], "payloads": {}, "multivariate": None}, + } + + ScheduledChange.objects.create( + team=self.team, + record_id=feature_flag.id, + model_name="FeatureFlag", + payload=payload, + scheduled_at=(datetime.now(timezone.utc) - timedelta(seconds=30)), + ) + + process_scheduled_changes() + + updated_flag = FeatureFlag.objects.get(key="flag-1") + self.assertEqual(updated_flag.filters["groups"][0], new_release_condition) + + def test_schedule_feature_flag_invalid_payload(self) -> None: + feature_flag = FeatureFlag.objects.create( + name="Flag 1", + key="flag-1", + active=False, + filters={"groups": []}, + team=self.team, + created_by=self.user, + ) + + payload = {"foo": "bar"} + + scheduled_change = ScheduledChange.objects.create( + team=self.team, + record_id=feature_flag.id, + model_name="FeatureFlag", + payload=payload, + scheduled_at=(datetime.now(timezone.utc) - timedelta(seconds=30)), + ) + + process_scheduled_changes() + + updated_flag = FeatureFlag.objects.get(key="flag-1") + self.assertEqual(updated_flag.filters["groups"], []) + + updated_scheduled_change = ScheduledChange.objects.get(id=scheduled_change.id) + self.assertEqual(updated_scheduled_change.failure_reason, "Invalid payload") + + @snapshot_postgres_queries + @freeze_time("2023-12-21T09:00:00Z") + def test_schedule_feature_flag_multiple_changes(self) -> None: + feature_flag = FeatureFlag.objects.create( + name="Flag", + key="flag-1", + active=True, + filters={"groups": []}, + team=self.team, + created_by=self.user, + ) + + # Create 4 scheduled changes + # 1. Due in the past + change_past_condition = { + "properties": [{"key": "$geoip_city_name", "value": ["Sydney"], "operator": "exact", "type": "person"}], + "rollout_percentage": 50, + "variant": None, + } + change_past = ScheduledChange.objects.create( + team=self.team, + record_id=feature_flag.id, + model_name="FeatureFlag", + payload={ + "operation": "add_release_condition", + "value": {"groups": [change_past_condition], "multivariate": None, "payloads": {}}, + }, + scheduled_at=(datetime.now(timezone.utc) - timedelta(hours=1)), + ) + + # 2. Due in the past and already executed + change_past_executed_at = datetime.now(timezone.utc) - timedelta(hours=5) + change_past_executed = ScheduledChange.objects.create( + team=self.team, + record_id=feature_flag.id, + model_name="FeatureFlag", + payload={"operation": "update_status", "value": False}, + scheduled_at=change_past_executed_at, + executed_at=change_past_executed_at, + ) + + # 3. Due exactly now + change_due_now_condition = { + "properties": [{"key": "$geoip_city_name", "value": ["New York"], "operator": "exact", "type": "person"}], + "rollout_percentage": 75, + "variant": None, + } + change_due_now = ScheduledChange.objects.create( + team=self.team, + record_id=feature_flag.id, + model_name="FeatureFlag", + payload={ + "operation": "add_release_condition", + "value": {"groups": [change_due_now_condition], "multivariate": None, "payloads": {}}, + }, + scheduled_at=datetime.now(timezone.utc), + ) + + # 4. Due in the future + change_due_future = ScheduledChange.objects.create( + team=self.team, + record_id=feature_flag.id, + model_name="FeatureFlag", + payload={"operation": "update_status", "value": False}, + scheduled_at=(datetime.now(timezone.utc) + timedelta(hours=1)), + ) + + process_scheduled_changes() + + # Refresh change records + change_past = ScheduledChange.objects.get(id=change_past.id) + change_past_executed = ScheduledChange.objects.get(id=change_past_executed.id) + change_due_now = ScheduledChange.objects.get(id=change_due_now.id) + change_due_future = ScheduledChange.objects.get(id=change_due_future.id) + + # Changes due have been marked executed + self.assertIsNotNone(change_past.executed_at) + self.assertIsNotNone(change_due_now.executed_at) + + # Other changes have not been executed + self.assertEqual(change_past_executed.executed_at, change_past_executed_at) + self.assertIsNone(change_due_future.executed_at) + + # The changes due have been propagated in the correct order (oldest scheduled_at first) + updated_flag = FeatureFlag.objects.get(key="flag-1") + self.assertEqual(updated_flag.filters["groups"], [change_past_condition, change_due_now_condition]) diff --git a/posthog/temporal/batch_exports/bigquery_batch_export.py b/posthog/temporal/batch_exports/bigquery_batch_export.py index b40a13bc83345..c802cc3192afe 100644 --- a/posthog/temporal/batch_exports/bigquery_batch_export.py +++ b/posthog/temporal/batch_exports/bigquery_batch_export.py @@ -223,7 +223,7 @@ async def flush_to_bigquery(): for field in table_schema if field.name != "bq_ingested_timestamp" } - row["bq_ingested_timestamp"] = str(dt.datetime.utcnow()) + row["bq_ingested_timestamp"] = str(dt.datetime.now(dt.timezone.utc)) jsonl_file.write_records_to_jsonl([row]) diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py index 0648ed01df59e..fc75f6beed4de 100644 --- a/posthog/temporal/data_imports/external_data_job.py +++ b/posthog/temporal/data_imports/external_data_job.py @@ -9,12 +9,10 @@ # TODO: remove dependency from posthog.temporal.batch_exports.base import PostHogWorkflow -from posthog.temporal.data_imports.pipelines.stripe.stripe_pipeline import ( - PIPELINE_TYPE_INPUTS_MAPPING, - PIPELINE_TYPE_RUN_MAPPING, - PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING, -) + from posthog.warehouse.data_load.validate_schema import validate_schema_and_update_table +from posthog.temporal.data_imports.pipelines.schemas import PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING +from posthog.temporal.data_imports.pipelines.pipeline import DataImportPipeline, PipelineInputs from posthog.warehouse.external_data_source.jobs import ( create_external_data_job, get_external_data_job, @@ -47,6 +45,8 @@ async def create_external_data_job_model(inputs: CreateExternalDataJobInputs) -> source = await sync_to_async(ExternalDataSource.objects.get)( # type: ignore team_id=inputs.team_id, id=inputs.external_data_source_id ) + source.status = "Running" + await sync_to_async(source.save)() # type: ignore # Sync schemas if they have changed await sync_to_async(sync_old_schemas_with_new_schemas)( # type: ignore @@ -133,19 +133,29 @@ async def run_external_data_job(inputs: ExternalDataJobInputs) -> None: team_id=inputs.team_id, run_id=inputs.run_id, ) + logger = await bind_temporal_worker_logger(team_id=inputs.team_id) - job_inputs = PIPELINE_TYPE_INPUTS_MAPPING[model.pipeline.source_type]( + job_inputs = PipelineInputs( source_id=inputs.source_id, schemas=inputs.schemas, run_id=inputs.run_id, team_id=inputs.team_id, job_type=model.pipeline.source_type, dataset_name=model.folder_path, - **model.pipeline.job_inputs, ) - job_fn = PIPELINE_TYPE_RUN_MAPPING[model.pipeline.source_type] - await job_fn(job_inputs) + source = None + if model.pipeline.source_type == ExternalDataSource.Type.STRIPE: + from posthog.temporal.data_imports.pipelines.stripe.helpers import stripe_source + + stripe_secret_key = model.pipeline.job_inputs.get("stripe_secret_key", None) + if not stripe_secret_key: + raise ValueError(f"Stripe secret key not found for job {model.id}") + source = stripe_source(api_key=stripe_secret_key, endpoints=tuple(inputs.schemas)) + else: + raise ValueError(f"Source type {model.pipeline.source_type} not supported") + + await DataImportPipeline(job_inputs, source, logger).run() # TODO: update retry policies diff --git a/posthog/temporal/data_imports/pipelines/pipeline.py b/posthog/temporal/data_imports/pipelines/pipeline.py new file mode 100644 index 0000000000000..ad6d53aa3a9e6 --- /dev/null +++ b/posthog/temporal/data_imports/pipelines/pipeline.py @@ -0,0 +1,89 @@ +from dataclasses import dataclass +from uuid import UUID + +import dlt +from django.conf import settings +from dlt.pipeline.exceptions import PipelineStepFailed + +import asyncio +import os +from posthog.settings.base_variables import TEST +from structlog.typing import FilteringBoundLogger +from dlt.sources import DltResource + + +@dataclass +class PipelineInputs: + source_id: UUID + run_id: str + schemas: list[str] + dataset_name: str + job_type: str + team_id: int + + +class DataImportPipeline: + loader_file_format = "parquet" + + def __init__(self, inputs: PipelineInputs, source: DltResource, logger: FilteringBoundLogger): + self.inputs = inputs + self.logger = logger + self.source = source + + def _get_pipeline_name(self): + return f"{self.inputs.job_type}_pipeline_{self.inputs.team_id}_run_{self.inputs.run_id}" + + def _get_pipelines_dir(self): + return f"{os.getcwd()}/.dlt/{self.inputs.team_id}/{self.inputs.run_id}/{self.inputs.job_type}" + + def _get_destination(self): + if TEST: + credentials = { + "aws_access_key_id": settings.AIRBYTE_BUCKET_KEY, + "aws_secret_access_key": settings.AIRBYTE_BUCKET_SECRET, + "endpoint_url": settings.OBJECT_STORAGE_ENDPOINT, + } + else: + credentials = { + "aws_access_key_id": settings.AIRBYTE_BUCKET_KEY, + "aws_secret_access_key": settings.AIRBYTE_BUCKET_SECRET, + } + + return dlt.destinations.filesystem( + credentials=credentials, + bucket_url=settings.BUCKET_URL, # type: ignore + ) + + def _create_pipeline(self): + pipeline_name = self._get_pipeline_name() + pipelines_dir = self._get_pipelines_dir() + destination = self._get_destination() + + return dlt.pipeline( + pipeline_name=pipeline_name, + pipelines_dir=pipelines_dir, + destination=destination, + dataset_name=self.inputs.dataset_name, + ) + + def _get_schemas(self): + if not self.inputs.schemas: + self.logger.info(f"No schemas found for source id {self.inputs.source_id}") + return None + + return self.inputs.schemas + + def _run(self): + pipeline = self._create_pipeline() + pipeline.run(self.source, loader_file_format=self.loader_file_format) + + async def run(self) -> None: + schemas = self._get_schemas() + if not schemas: + return + + try: + await asyncio.to_thread(self._run) + except PipelineStepFailed: + self.logger.error(f"Data import failed for endpoint") + raise diff --git a/posthog/temporal/data_imports/pipelines/schemas.py b/posthog/temporal/data_imports/pipelines/schemas.py new file mode 100644 index 0000000000000..a62db7d664e40 --- /dev/null +++ b/posthog/temporal/data_imports/pipelines/schemas.py @@ -0,0 +1,4 @@ +from posthog.warehouse.models import ExternalDataSource +from posthog.temporal.data_imports.pipelines.stripe.settings import ENDPOINTS + +PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING = {ExternalDataSource.Type.STRIPE: ENDPOINTS} diff --git a/posthog/temporal/data_imports/pipelines/stripe/stripe_pipeline.py b/posthog/temporal/data_imports/pipelines/stripe/stripe_pipeline.py deleted file mode 100644 index a1138c74aa10e..0000000000000 --- a/posthog/temporal/data_imports/pipelines/stripe/stripe_pipeline.py +++ /dev/null @@ -1,90 +0,0 @@ -from dataclasses import dataclass -from typing import Dict -from uuid import UUID - -import dlt -from django.conf import settings -from dlt.pipeline.exceptions import PipelineStepFailed - -from posthog.warehouse.models import ExternalDataSource -from posthog.temporal.data_imports.pipelines.stripe.helpers import stripe_source -from posthog.temporal.data_imports.pipelines.stripe.settings import ENDPOINTS -from posthog.temporal.common.logger import bind_temporal_worker_logger -import asyncio -import os -from posthog.settings.base_variables import TEST - - -@dataclass -class PipelineInputs: - source_id: UUID - run_id: str - schemas: list[str] - dataset_name: str - job_type: str - team_id: int - - -@dataclass -class SourceColumnType: - name: str - data_type: str - nullable: bool - - -@dataclass -class SourceSchema: - resource: str - name: str - columns: Dict[str, SourceColumnType] - write_disposition: str - - -@dataclass -class StripeJobInputs(PipelineInputs): - stripe_secret_key: str - - -def create_pipeline(inputs: PipelineInputs): - pipeline_name = f"{inputs.job_type}_pipeline_{inputs.team_id}_run_{inputs.run_id}" - pipelines_dir = f"{os.getcwd()}/.dlt/{inputs.team_id}/{inputs.run_id}/{inputs.job_type}" - - return dlt.pipeline( - pipeline_name=pipeline_name, - pipelines_dir=pipelines_dir, - destination=dlt.destinations.filesystem( - credentials={ - "aws_access_key_id": settings.AIRBYTE_BUCKET_KEY, - "aws_secret_access_key": settings.AIRBYTE_BUCKET_SECRET, - "endpoint_url": settings.OBJECT_STORAGE_ENDPOINT if TEST else None, - }, - bucket_url=settings.BUCKET_URL, # type: ignore - ), - dataset_name=inputs.dataset_name, - ) - - -def _run_pipeline(inputs: StripeJobInputs): - pipeline = create_pipeline(inputs) - source = stripe_source(inputs.stripe_secret_key, tuple(inputs.schemas)) - pipeline.run(source, loader_file_format="parquet") - - -# a temporal activity -async def run_stripe_pipeline(inputs: StripeJobInputs) -> None: - logger = await bind_temporal_worker_logger(team_id=inputs.team_id) - schemas = inputs.schemas - if not schemas: - logger.info(f"No schemas found for source id {inputs.source_id}") - return - - try: - await asyncio.to_thread(_run_pipeline, inputs) - except PipelineStepFailed: - logger.error(f"Data import failed for endpoint") - raise - - -PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING = {ExternalDataSource.Type.STRIPE: ENDPOINTS} -PIPELINE_TYPE_INPUTS_MAPPING = {ExternalDataSource.Type.STRIPE: StripeJobInputs} -PIPELINE_TYPE_RUN_MAPPING = {ExternalDataSource.Type.STRIPE: run_stripe_pipeline} diff --git a/posthog/temporal/tests/batch_exports/test_backfill_batch_export.py b/posthog/temporal/tests/batch_exports/test_backfill_batch_export.py index 436b76dfb3876..dc5ef36c0b5f0 100644 --- a/posthog/temporal/tests/batch_exports/test_backfill_batch_export.py +++ b/posthog/temporal/tests/batch_exports/test_backfill_batch_export.py @@ -261,7 +261,7 @@ async def test_backfill_batch_export_workflow_fails_when_schedule_deleted_after_ """ start_at = dt.datetime(2023, 1, 1, 0, 0, 0, tzinfo=dt.timezone.utc) end_at = dt.datetime(2023, 1, 1, 0, 10, 0, tzinfo=dt.timezone.utc) - now = dt.datetime.utcnow() + now = dt.datetime.now(dt.timezone.utc) desc = await temporal_schedule.describe() diff --git a/posthog/temporal/tests/batch_exports/test_bigquery_batch_export_workflow.py b/posthog/temporal/tests/batch_exports/test_bigquery_batch_export_workflow.py index 6ab3239edbeec..b106b814ea3e2 100644 --- a/posthog/temporal/tests/batch_exports/test_bigquery_batch_export_workflow.py +++ b/posthog/temporal/tests/batch_exports/test_bigquery_batch_export_workflow.py @@ -41,8 +41,7 @@ pytestmark = [SKIP_IF_MISSING_GOOGLE_APPLICATION_CREDENTIALS, pytest.mark.asyncio, pytest.mark.django_db] - -TEST_TIME = dt.datetime.utcnow() +TEST_TIME = dt.datetime.now(dt.timezone.utc) def assert_events_in_bigquery( diff --git a/posthog/temporal/tests/batch_exports/test_logger.py b/posthog/temporal/tests/batch_exports/test_logger.py index b3e2611979363..a3ceb011e27a2 100644 --- a/posthog/temporal/tests/batch_exports/test_logger.py +++ b/posthog/temporal/tests/batch_exports/test_logger.py @@ -211,13 +211,13 @@ def activity_environment(request): "activity_environment", [ ActivityInfo( - workflow_id=f"{BATCH_EXPORT_ID}-{dt.datetime.utcnow()}", + workflow_id=f"{BATCH_EXPORT_ID}-{dt.datetime.now(dt.timezone.utc)}", workflow_type="s3-export", workflow_run_id=str(uuid.uuid4()), attempt=random.randint(1, 10000), ), ActivityInfo( - workflow_id=f"{BATCH_EXPORT_ID}-Backfill-{dt.datetime.utcnow()}", + workflow_id=f"{BATCH_EXPORT_ID}-Backfill-{dt.datetime.now(dt.timezone.utc)}", workflow_type="backfill-batch-export", workflow_run_id=str(uuid.uuid4()), attempt=random.randint(1, 10000), @@ -262,13 +262,13 @@ async def log_activity(): "activity_environment", [ ActivityInfo( - workflow_id=f"{BATCH_EXPORT_ID}-{dt.datetime.utcnow()}", + workflow_id=f"{BATCH_EXPORT_ID}-{dt.datetime.now(dt.timezone.utc)}", workflow_type="s3-export", workflow_run_id=str(uuid.uuid4()), attempt=random.randint(1, 10000), ), ActivityInfo( - workflow_id=f"{BATCH_EXPORT_ID}-Backfill-{dt.datetime.utcnow()}", + workflow_id=f"{BATCH_EXPORT_ID}-Backfill-{dt.datetime.now(dt.timezone.utc)}", workflow_type="backfill-batch-export", workflow_run_id=str(uuid.uuid4()), attempt=random.randint(1, 10000), @@ -324,13 +324,13 @@ def log_entries_table(): "activity_environment", [ ActivityInfo( - workflow_id=f"{BATCH_EXPORT_ID}-{dt.datetime.utcnow()}", + workflow_id=f"{BATCH_EXPORT_ID}-{dt.datetime.now(dt.timezone.utc)}", workflow_type="s3-export", workflow_run_id=str(uuid.uuid4()), attempt=random.randint(1, 10000), ), ActivityInfo( - workflow_id=f"{BATCH_EXPORT_ID}-Backfill-{dt.datetime.utcnow()}", + workflow_id=f"{BATCH_EXPORT_ID}-Backfill-{dt.datetime.now(dt.timezone.utc)}", workflow_type="backfill-batch-export", workflow_run_id=str(uuid.uuid4()), attempt=random.randint(1, 10000), diff --git a/posthog/temporal/tests/test_external_data_job.py b/posthog/temporal/tests/test_external_data_job.py index e519b334693f0..1af196f368831 100644 --- a/posthog/temporal/tests/test_external_data_job.py +++ b/posthog/temporal/tests/test_external_data_job.py @@ -28,10 +28,10 @@ ExternalDataSchema, ) -from posthog.temporal.data_imports.pipelines.stripe.stripe_pipeline import ( - PIPELINE_TYPE_RUN_MAPPING, +from posthog.temporal.data_imports.pipelines.schemas import ( PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING, ) +from posthog.temporal.data_imports.pipelines.pipeline import DataImportPipeline from temporalio.testing import WorkflowEnvironment from temporalio.common import RetryPolicy from temporalio.worker import UnsandboxedWorkflowRunner, Worker @@ -449,7 +449,7 @@ async def mock_async_func(inputs): with mock.patch( "posthog.warehouse.models.table.DataWarehouseTable.get_columns", return_value={"id": "string"} - ), mock.patch.dict(PIPELINE_TYPE_RUN_MAPPING, {ExternalDataSource.Type.STRIPE: mock_async_func}): + ), mock.patch.object(DataImportPipeline, "run", mock_async_func): with override_settings(AIRBYTE_BUCKET_KEY="test-key", AIRBYTE_BUCKET_SECRET="test-secret"): async with await WorkflowEnvironment.start_time_skipping() as activity_environment: async with Worker( diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py index 4dadbd33ab7fc..c72cbd14e6c9d 100644 --- a/posthog/warehouse/api/external_data_source.py +++ b/posthog/warehouse/api/external_data_source.py @@ -21,7 +21,7 @@ ) from posthog.warehouse.models import ExternalDataSource, ExternalDataSchema, ExternalDataJob from posthog.warehouse.api.external_data_schema import ExternalDataSchemaSerializer -from posthog.temporal.data_imports.pipelines.stripe.stripe_pipeline import ( +from posthog.temporal.data_imports.pipelines.schemas import ( PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING, ) diff --git a/posthog/warehouse/api/test/test_external_data_source.py b/posthog/warehouse/api/test/test_external_data_source.py index f05ade40513c3..2ad741b453a29 100644 --- a/posthog/warehouse/api/test/test_external_data_source.py +++ b/posthog/warehouse/api/test/test_external_data_source.py @@ -2,7 +2,7 @@ from posthog.warehouse.models import ExternalDataSource, ExternalDataSchema import uuid from unittest.mock import patch -from posthog.temporal.data_imports.pipelines.stripe.stripe_pipeline import ( +from posthog.temporal.data_imports.pipelines.schemas import ( PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING, ) diff --git a/posthog/warehouse/api/test/test_view_link.py b/posthog/warehouse/api/test/test_view_link.py index d7a265d9c159e..ba01536ded421 100644 --- a/posthog/warehouse/api/test/test_view_link.py +++ b/posthog/warehouse/api/test/test_view_link.py @@ -216,8 +216,8 @@ def test_view_link_nested_multiple_joins(self): self.assertEqual( query_response["types"], [ - ("events__event_view.fake", "String"), - ("events__person_view.p_distinct_id", "String"), + ("fake", "String"), + ("p_distinct_id", "String"), ], )