From 4a3a9e6dfdd91c8f6adbb7841a39b203b15b367d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 25 Apr 2024 16:14:47 +0200 Subject: [PATCH 1/8] fix: Default to 0 if nothing to sum in batch exports app metrics (#21851) * fix: Default to 0 if nothing to sum * test: Add test case --- posthog/api/app_metrics.py | 8 +++- posthog/api/test/test_app_metrics.py | 69 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/posthog/api/app_metrics.py b/posthog/api/app_metrics.py index 7107f76a443c5..3afdda18ef853 100644 --- a/posthog/api/app_metrics.py +++ b/posthog/api/app_metrics.py @@ -118,8 +118,12 @@ def get_batch_export_runs_app_metrics_queryset(self, batch_export_id: str): .annotate(dates=TruncDay("last_updated_at")) .values("dates") .annotate( - successes=Sum(Coalesce("records_total_count", 0), filter=Q(status=BatchExportRun.Status.COMPLETED)), - failures=Sum(Coalesce("records_total_count", 0), filter=~Q(status=BatchExportRun.Status.COMPLETED)), + successes=Sum( + Coalesce("records_total_count", 0), filter=Q(status=BatchExportRun.Status.COMPLETED), default=0 + ), + failures=Sum( + Coalesce("records_total_count", 0), filter=~Q(status=BatchExportRun.Status.COMPLETED), default=0 + ), ) .order_by("dates") .all() diff --git a/posthog/api/test/test_app_metrics.py b/posthog/api/test/test_app_metrics.py index 4659d1c023a31..dd9c01ba023b6 100644 --- a/posthog/api/test/test_app_metrics.py +++ b/posthog/api/test/test_app_metrics.py @@ -156,6 +156,75 @@ def test_retrieve_batch_export_runs_app_metrics(self): }, ) + def test_retrieve_batch_export_runs_app_metrics_defaults_to_zero(self): + """Test batch export metrics returned by app metrics endpoint.""" + destination_data = { + "type": "S3", + "config": { + "bucket_name": "my-production-s3-bucket", + "region": "us-east-1", + "prefix": "posthog-events/", + "aws_access_key_id": "abc123", + "aws_secret_access_key": "secret", + }, + } + + batch_export_data = { + "name": "my-production-s3-bucket-destination", + "destination": destination_data, + "interval": "hour", + } + + temporal = sync_connect() + now = dt.datetime(2021, 12, 5, 13, 23, 0, tzinfo=dt.timezone.utc) + + with start_test_worker(temporal): + response = create_batch_export_ok( + self.client, + self.team.pk, + json.dumps(batch_export_data), + ) + batch_export_id = response["id"] + + for days_ago in range(0, 7): + last_updated_at = now - dt.timedelta(days=days_ago) + + with freeze_time(last_updated_at): + # Since 'last_updated_at' uses 'auto_now', passing the argument is ignored. + # We have to re-freeze time to get each run created on a single date. + BatchExportRun.objects.create( + batch_export_id=batch_export_id, + data_interval_end=last_updated_at, + data_interval_start=last_updated_at - dt.timedelta(hours=1), + status=BatchExportRun.Status.COMPLETED, + records_completed=1, + records_total_count=1, + ) + + response = self.client.get(f"/api/projects/@current/app_metrics/{batch_export_id}?date_from=-7d") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + response.json(), + { + "metrics": { + "dates": [ + "2021-11-29", + "2021-11-30", + "2021-12-01", + "2021-12-02", + "2021-12-03", + "2021-12-04", + "2021-12-05", + ], + "successes": [1, 1, 1, 1, 1, 1, 1], + "successes_on_retry": [0, 0, 0, 0, 0, 0, 0], + "failures": [0, 0, 0, 0, 0, 0, 0], + "totals": {"successes": 7, "successes_on_retry": 0, "failures": 0}, + }, + "errors": None, + }, + ) + def test_list_historical_exports(self): self._create_activity_log( activity="job_triggered", From 85351860d299a5b55f54e8dcc5c4b6fab1e97a95 Mon Sep 17 00:00:00 2001 From: Ben White Date: Thu, 25 Apr 2024 16:16:02 +0200 Subject: [PATCH 2/8] chore: Remove heatmaps preview config option (#21846) --- frontend/src/loadPostHogJS.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/src/loadPostHogJS.tsx b/frontend/src/loadPostHogJS.tsx index a45ccdca06dbd..ed96f7dfccc32 100644 --- a/frontend/src/loadPostHogJS.tsx +++ b/frontend/src/loadPostHogJS.tsx @@ -44,8 +44,6 @@ export function loadPostHogJS(): void { }, process_person: 'identified_only', - __preview_heatmaps: true, - // Helper to capture events for assertions in Cypress _onCapture: (window as any)._cypress_posthog_captures ? (_, event) => (window as any)._cypress_posthog_captures.push(event) From e7e3514b412efc923d955db6300b3e6f9c8b739f Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 25 Apr 2024 17:04:03 +0200 Subject: [PATCH 3/8] chore(capture): allow to skip sha hashing of the kafka messages (#21850) --- posthog/api/capture.py | 6 +++++- posthog/settings/ingestion.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/posthog/api/capture.py b/posthog/api/capture.py index 9c223f8264acb..aaa3998213ce8 100644 --- a/posthog/api/capture.py +++ b/posthog/api/capture.py @@ -625,7 +625,11 @@ def capture_internal( ): kafka_partition_key = None else: - kafka_partition_key = hashlib.sha256(candidate_partition_key.encode()).hexdigest() + if settings.CAPTURE_SKIP_KEY_HASHING: + kafka_partition_key = candidate_partition_key + else: + # TODO: remove after progressive rollout of the option + kafka_partition_key = hashlib.sha256(candidate_partition_key.encode()).hexdigest() return log_event(parsed_event, event["event"], partition_key=kafka_partition_key, historical=historical) diff --git a/posthog/settings/ingestion.py b/posthog/settings/ingestion.py index 8a3a24d981ce0..7f6296e559114 100644 --- a/posthog/settings/ingestion.py +++ b/posthog/settings/ingestion.py @@ -17,6 +17,9 @@ # partitioning-related settings below. CAPTURE_ALLOW_RANDOM_PARTITIONING = get_from_env("CAPTURE_ALLOW_RANDOM_PARTITIONING", True, type_cast=str_to_bool) +# TOOD: make default after rollout on both prods: remove the superfluous hashing of the Kafka message key +CAPTURE_SKIP_KEY_HASHING = get_from_env("CAPTURE_SKIP_KEY_HASHING", type_cast=bool, default=False) + # A list of pairs (in the format 2:myLovelyId) that we should use # random partitioning for when producing events to the Kafka topic consumed by the plugin server. # This is a measure to handle hot partitions in ad-hoc cases. From d2ed2ea9dfa359856356a2e38cc961b9c10d60d2 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 25 Apr 2024 17:05:01 +0200 Subject: [PATCH 4/8] fix: Wrap asgi application to ignore lifetime requests (#21802) Wrap asgi application to ignore lifetime requests ASGI servers send "lifetime" requests to django at startup and shutdown. The ASGI spec allows these to fail, in which case the server ignores them, however the way django fails them sends an exception to sentry which we don't want. Overwrite the handler to return a 501 error without throwing an exception --- posthog/asgi.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/posthog/asgi.py b/posthog/asgi.py index 22912a0c7b76e..678645bb01d3b 100644 --- a/posthog/asgi.py +++ b/posthog/asgi.py @@ -1,8 +1,17 @@ import os from django.core.asgi import get_asgi_application +from django.http.response import HttpResponse os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings") os.environ.setdefault("SERVER_GATEWAY_INTERFACE", "ASGI") -application = get_asgi_application() + +def lifetime_wrapper(func): + async def inner(scope, receive, send): + if scope["type"] != "http": + return HttpResponse(status=501) + return func(scope, receive, send) + + +application = lifetime_wrapper(get_asgi_application()) From 3a7646b49f85bc6baf9588fa694d58a391ea2cfd Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 25 Apr 2024 16:08:14 +0100 Subject: [PATCH 5/8] fix: same numbers everywhere (#21848) --- posthog/heatmaps/heatmaps_api.py | 12 +- .../test/__snapshots__/test_heatmaps_api.ambr | 146 +++++++-------- posthog/heatmaps/test/test_heatmaps_api.py | 176 +++++++++++------- 3 files changed, 188 insertions(+), 146 deletions(-) diff --git a/posthog/heatmaps/heatmaps_api.py b/posthog/heatmaps/heatmaps_api.py index f06899e3c4178..e3fa68b5b4db3 100644 --- a/posthog/heatmaps/heatmaps_api.py +++ b/posthog/heatmaps/heatmaps_api.py @@ -17,17 +17,17 @@ from posthog.utils import relative_date_parse_with_delta_mapping DEFAULT_QUERY = """ - select pointer_target_fixed, relative_client_x, client_y, {aggregation_count} + select pointer_target_fixed, pointer_relative_x, client_y, {aggregation_count} from ( select distinct_id, pointer_target_fixed, - round((x / viewport_width), 2) as relative_client_x, + round((x / viewport_width), 2) as pointer_relative_x, y * scale_factor as client_y from heatmaps where {predicates} ) - group by `pointer_target_fixed`, relative_client_x, client_y + group by `pointer_target_fixed`, pointer_relative_x, client_y """ SCROLL_DEPTH_QUERY = """ @@ -106,7 +106,7 @@ class HeatmapsResponseSerializer(serializers.Serializer): class HeatmapScrollDepthResponseItemSerializer(serializers.Serializer): cumulative_count = serializers.IntegerField(required=True) bucket_count = serializers.IntegerField(required=True) - scroll_depth_bucket = serializers.FloatField(required=True) + scroll_depth_bucket = serializers.IntegerField(required=True) class HeatmapsScrollDepthResponseSerializer(serializers.Serializer): @@ -163,8 +163,8 @@ def _predicate_expressions(placeholders: dict[str, Expr]) -> List[ast.Expr]: # "type": "`type` = {type}", # optional "date_to": "timestamp <= {date_to} + interval 1 day", - "viewport_width_min": "viewport_width >= ceil({viewport_width_min} / 16)", - "viewport_width_max": "viewport_width <= ceil({viewport_width_max} / 16)", + "viewport_width_min": "viewport_width >= round({viewport_width_min} / 16)", + "viewport_width_max": "viewport_width <= round({viewport_width_max} / 16)", "url_exact": "current_url = {url_exact}", "url_pattern": "match(current_url, {url_pattern})", } diff --git a/posthog/heatmaps/test/__snapshots__/test_heatmaps_api.ambr b/posthog/heatmaps/test/__snapshots__/test_heatmaps_api.ambr index 1f5fe0431add1..faa7d77c2c79a 100644 --- a/posthog/heatmaps/test/__snapshots__/test_heatmaps_api.ambr +++ b/posthog/heatmaps/test/__snapshots__/test_heatmaps_api.ambr @@ -3,18 +3,18 @@ ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'rageclick'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')), equals(heatmaps.current_url, 'http://example.com'))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, @@ -25,18 +25,18 @@ ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'rageclick'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')), equals(heatmaps.current_url, 'http://example.com/about'))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, @@ -47,282 +47,282 @@ ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'rageclick'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')), match(heatmaps.current_url, 'http://example.com*'))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_all_data_response +# name: TestSessionRecordings.test_can_filter_by_viewport_0_min_150 ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, round(divide(150, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_count_by_aggregation +# name: TestSessionRecordings.test_can_filter_by_viewport_1_min_161 ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, round(divide(161, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_count_by_aggregation.1 +# name: TestSessionRecordings.test_can_filter_by_viewport_2_min_177 ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, - count(DISTINCT distinct_id) AS cnt + count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, round(divide(177, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_empty_response +# name: TestSessionRecordings.test_can_filter_by_viewport_3_min_201 ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2024-05-03')))) + WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, round(divide(201, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_filter_by_click +# name: TestSessionRecordings.test_can_filter_by_viewport_4_min_161_and_max_192 ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, round(divide(161, 16))), 0), ifNull(lessOrEquals(heatmaps.viewport_width, round(divide(192, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_filter_by_click.1 +# name: TestSessionRecordings.test_can_get_all_data_response ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'rageclick'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_filter_by_date_from +# name: TestSessionRecordings.test_can_get_count_by_aggregation ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_filter_by_min_and_max_viewport +# name: TestSessionRecordings.test_can_get_count_by_aggregation.1 ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, - count(*) AS cnt + count(DISTINCT distinct_id) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, ceil(divide(161, 16))), 0), ifNull(lessOrEquals(heatmaps.viewport_width, ceil(divide(192, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_filter_by_min_viewport +# name: TestSessionRecordings.test_can_get_empty_response ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, ceil(divide(150, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2024-05-03')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_filter_by_min_viewport.1 +# name: TestSessionRecordings.test_can_get_filter_by_click ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, ceil(divide(161, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_filter_by_min_viewport.2 +# name: TestSessionRecordings.test_can_get_filter_by_click.1 ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, ceil(divide(177, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'rageclick'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 ''' # --- -# name: TestSessionRecordings.test_can_get_filter_by_min_viewport.3 +# name: TestSessionRecordings.test_can_get_filter_by_date_from ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps - WHERE and(equals(heatmaps.team_id, 2), ifNull(greaterOrEquals(heatmaps.viewport_width, ceil(divide(193, 16))), 0), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) + WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, @@ -333,18 +333,18 @@ ''' /* user_id:0 request:_snapshot_ */ SELECT pointer_target_fixed AS pointer_target_fixed, - relative_client_x AS relative_client_x, + pointer_relative_x AS pointer_relative_x, client_y AS client_y, count(*) AS cnt FROM (SELECT heatmaps.distinct_id AS distinct_id, heatmaps.pointer_target_fixed AS pointer_target_fixed, - round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS relative_client_x, + round(divide(heatmaps.x, heatmaps.viewport_width), 2) AS pointer_relative_x, multiply(heatmaps.y, heatmaps.scale_factor) AS client_y FROM heatmaps WHERE and(equals(heatmaps.team_id, 2), equals(heatmaps.type, 'click'), greaterOrEquals(toTimeZone(heatmaps.timestamp, 'UTC'), toDate('2023-03-08')))) GROUP BY pointer_target_fixed, - relative_client_x, + pointer_relative_x, client_y LIMIT 1000000 SETTINGS readonly=2, max_execution_time=60, diff --git a/posthog/heatmaps/test/test_heatmaps_api.py b/posthog/heatmaps/test/test_heatmaps_api.py index e07343a2760b4..18a2c2205d4e6 100644 --- a/posthog/heatmaps/test/test_heatmaps_api.py +++ b/posthog/heatmaps/test/test_heatmaps_api.py @@ -1,5 +1,3 @@ -import math - import freezegun from django.http import HttpResponse from parameterized import parameterized @@ -145,12 +143,21 @@ def test_can_filter_by_exact_url(self) -> None: @snapshot_clickhouse_queries def test_can_get_scrolldepth_counts(self) -> None: + # to calculate expected scroll depth bucket from y and viewport height + # ((round(y/16) + round(viewport_height/16)) * 16 // 100) * 100 + + # scroll depth bucket 1000 self._create_heatmap_event("session_1", "scrolldepth", "2023-03-08T07:00:00", y=10, viewport_height=1000) self._create_heatmap_event("session_2", "scrolldepth", "2023-03-08T08:00:00", y=100, viewport_height=1000) + # scroll depth bucket 1100 self._create_heatmap_event("session_3", "scrolldepth", "2023-03-08T08:01:00", y=200, viewport_height=1000) + # scroll depth bucket 1200 self._create_heatmap_event("session_4", "scrolldepth", "2023-03-08T08:01:00", y=300, viewport_height=1000) + # scroll depth bucket 1300 self._create_heatmap_event("session_5", "scrolldepth", "2023-03-08T08:01:00", y=400, viewport_height=1000) + # scroll depth bucket 1400 self._create_heatmap_event("session_6", "scrolldepth", "2023-03-08T08:01:00", y=500, viewport_height=1000) + # scroll depth bucket 1800 self._create_heatmap_event("session_7", "scrolldepth", "2023-03-08T08:01:00", y=900, viewport_height=1000) self._create_heatmap_event("session_8", "scrolldepth", "2023-03-08T08:01:00", y=900, viewport_height=1000) @@ -159,74 +166,139 @@ def test_can_get_scrolldepth_counts(self) -> None: assert scroll_response.json() == { "results": [ { - "bucket_count": 1, + "bucket_count": 2, "cumulative_count": 8, "scroll_depth_bucket": 1000, }, { "bucket_count": 1, - "cumulative_count": 7, + "cumulative_count": 6, "scroll_depth_bucket": 1100, }, { - "bucket_count": 2, - "cumulative_count": 6, + "bucket_count": 1, + "cumulative_count": 5, "scroll_depth_bucket": 1200, }, { "bucket_count": 1, "cumulative_count": 4, - "scroll_depth_bucket": 1400, + "scroll_depth_bucket": 1300, }, { "bucket_count": 1, "cumulative_count": 3, - "scroll_depth_bucket": 1500, + "scroll_depth_bucket": 1400, }, { "bucket_count": 2, "cumulative_count": 2, - "scroll_depth_bucket": 1900, + "scroll_depth_bucket": 1800, }, ], } - @snapshot_clickhouse_queries - def test_can_get_filter_by_min_viewport(self) -> None: - # all scale to 10 - self._create_heatmap_event("session_1", "click", "2023-03-08T08:00:00", 150) - self._create_heatmap_event("session_2", "click", "2023-03-08T08:00:00", 151) - self._create_heatmap_event("session_3", "click", "2023-03-08T08:01:00", 152) - # scale to 11 - self._create_heatmap_event("session_3", "click", "2023-03-08T08:01:00", 161) - # scales to 12 - self._create_heatmap_event("session_3", "click", "2023-03-08T08:01:00", 177) + def test_can_get_scrolldepth_counts_by_visitor(self) -> None: + # scroll depth bucket 1000 + self._create_heatmap_event( + "session_1", "scrolldepth", "2023-03-08T07:00:00", y=100, viewport_height=1000, distinct_id="12345" + ) - self._assert_heatmap_single_result_count({"date_from": "2023-03-08", "viewport_width_min": "150"}, 5) - self._assert_heatmap_single_result_count({"date_from": "2023-03-08", "viewport_width_min": "161"}, 2) - self._assert_heatmap_single_result_count({"date_from": "2023-03-08", "viewport_width_min": "177"}, 1) - self._assert_heatmap_no_result_count({"date_from": "2023-03-08", "viewport_width_min": "193"}) + # one person only scrolls a little way + # scroll depth bucket 1000 + self._create_heatmap_event( + "session_2", "scrolldepth", "2023-03-08T08:00:00", y=100, viewport_height=1000, distinct_id="34567" + ) + + # the first person scrolls further + # scroll depth bucket 1100 + self._create_heatmap_event( + "session_3", "scrolldepth", "2023-03-08T08:01:00", y=200, viewport_height=1000, distinct_id="12345" + ) + + scroll_response = self._get_heatmap( + {"date_from": "2023-03-06", "type": "scrolldepth", "aggregation": "unique_visitors"} + ) + + assert scroll_response.json() == { + "results": [ + { + "bucket_count": 2, + "cumulative_count": 3, + "scroll_depth_bucket": 1000, + }, + { + "bucket_count": 1, + "cumulative_count": 1, + "scroll_depth_bucket": 1100, + }, + ], + } + @staticmethod + def heatmap_result(relative_x: float, count: int) -> dict: + return { + "count": count, + "pointer_relative_x": relative_x, + "pointer_target_fixed": True, + "pointer_y": 16, + } + + @parameterized.expand( + [ + [ + "min_150", + {"date_from": "2023-03-08", "viewport_width_min": "150"}, + [heatmap_result(0.08, 1), heatmap_result(0.09, 1), heatmap_result(0.1, 1), heatmap_result(0.11, 2)], + ], + [ + "min_161", + {"date_from": "2023-03-08", "viewport_width_min": "161"}, + [ + heatmap_result(0.08, 1), + heatmap_result(0.09, 1), + heatmap_result(0.1, 1), + ], + ], + [ + "min_177", + {"date_from": "2023-03-08", "viewport_width_min": "177"}, + [ + heatmap_result(0.08, 1), + heatmap_result(0.09, 1), + ], + ], + ["min_201", {"date_from": "2023-03-08", "viewport_width_min": "201"}, []], + [ + "min_161_and_max_192", + {"date_from": "2023-03-08", "viewport_width_min": 161, "viewport_width_max": 192}, + [heatmap_result(0.08, 1), heatmap_result(0.09, 1), heatmap_result(0.1, 1)], + ], + ] + ) @snapshot_clickhouse_queries - def test_can_get_filter_by_min_and_max_viewport(self) -> None: - # all scale to 10 + def test_can_filter_by_viewport(self, _name: str, query_params: dict, expected_results: list) -> None: + # all these xs = round(10/16) = 1 + + # viewport widths that scale to 9 self._create_heatmap_event("session_1", "click", "2023-03-08T08:00:00", 150) self._create_heatmap_event("session_2", "click", "2023-03-08T08:00:00", 151) + + # viewport widths that scale to 10 self._create_heatmap_event("session_3", "click", "2023-03-08T08:01:00", 152) - # scale to 11 self._create_heatmap_event("session_3", "click", "2023-03-08T08:01:00", 161) - # scales to 12 + + # viewport width that scales to 11 self._create_heatmap_event("session_3", "click", "2023-03-08T08:01:00", 177) - # scales to 13 + # viewport width that scales to 12 self._create_heatmap_event("session_3", "click", "2023-03-08T08:01:00", 193) - self._assert_heatmap_single_result_count( - {"date_from": "2023-03-08", "viewport_width_min": 161, "viewport_width_max": 192}, 2 - ) + response = self._get_heatmap(query_params) + assert sorted(response.json()["results"], key=lambda k: k["pointer_relative_x"]) == expected_results @snapshot_clickhouse_queries def test_can_get_count_by_aggregation(self) -> None: - # 3 items but 2 viitors + # 3 items but 2 visitors self._create_heatmap_event("session_1", "click", distinct_id="12345") self._create_heatmap_event("session_2", "click", distinct_id="12345") self._create_heatmap_event("session_3", "click", distinct_id="54321") @@ -250,37 +322,6 @@ def test_only_allow_valid_values_for_aggregation(self, choice: str | None, expec {"date_from": "2023-03-08", "aggregation": choice}, expected_status_code=expected_status_code ) - def test_can_get_scrolldepth_counts_by_visitor(self) -> None: - self._create_heatmap_event( - "session_1", "scrolldepth", "2023-03-08T07:00:00", y=100, viewport_height=1000, distinct_id="12345" - ) - # one person only scrolls a little way - self._create_heatmap_event( - "session_2", "scrolldepth", "2023-03-08T08:00:00", y=100, viewport_height=1000, distinct_id="34567" - ) - self._create_heatmap_event( - "session_3", "scrolldepth", "2023-03-08T08:01:00", y=200, viewport_height=1000, distinct_id="12345" - ) - - scroll_response = self._get_heatmap( - {"date_from": "2023-03-06", "type": "scrolldepth", "aggregation": "unique_visitors"} - ) - - assert scroll_response.json() == { - "results": [ - { - "bucket_count": 2, - "cumulative_count": 3, - "scroll_depth_bucket": 1100, - }, - { - "bucket_count": 1, - "cumulative_count": 1, - "scroll_depth_bucket": 1200, - }, - ], - } - def _create_heatmap_event( self, session_id: str, @@ -288,6 +329,7 @@ def _create_heatmap_event( date_from: str = "2023-03-08T09:00:00", viewport_width: int = 100, viewport_height: int = 100, + x: int = 10, y: int = 20, current_url: str | None = None, distinct_id: str = "user_distinct_id", @@ -306,12 +348,12 @@ def _create_heatmap_event( "team_id": team_id, "distinct_id": distinct_id, "timestamp": format_clickhouse_timestamp(date_from), - "x": 10 / 16, - "y": y / 16, + "x": round(x / 16), + "y": round(y / 16), "scale_factor": 16, # this adjustment is done at ingestion - "viewport_width": math.ceil(viewport_width / 16), - "viewport_height": math.ceil(viewport_height / 16), + "viewport_width": round(viewport_width / 16), + "viewport_height": round(viewport_height / 16), "type": type, "pointer_target_fixed": True, "current_url": current_url if current_url else "http://posthog.com", From e3acb16cb264a8b3c8bdae4288bd279c921ebe8a Mon Sep 17 00:00:00 2001 From: Robbie Date: Thu, 25 Apr 2024 16:10:52 +0100 Subject: [PATCH 6/8] feat(web-analytics): Session filters 1 (#21512) * Handle hogql session properties * Add test for two lazy joins in one query (session & person) * Support passing session properties in web analytics queries * Fix typings * Working session definition fetching * Working session property definitions * Remove property allow list in web analytics * Put session properties first * Add session values * Rename duration back to $session_duration * Fix "keeps infiniteListCounts in sync" * Fix "setting search query filters events" * Fix taxonomy tests * Change session api to GenericViewSet * Add local properties back * Support datetime and boolean session properties * Update mypy baseline * Hide duration as a property and asterisk field * Add AsyncReturnType * Make taxonoicFilterLogic tests more reliable (maybe) * Fix duplicates from bad rebase * Only show session table session properties if feature flag enabled * Improve error message for non-hogql non-duration session property * Put session table properties behind FF * Revert taxonomicFilterLogic tests * Update query snapshots * Rename TaxonomicFilterGroupType.Sessions to TaxonomicFilterGroupType.SessionProperties * Use ViewSet instead of GenericViewSet * Add tests for session properties and session property values api * Formatting * New typing * Update query snapshots --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- frontend/src/lib/api.ts | 21 +++ .../DefinitionPopover/DefinitionPopover.tsx | 3 +- .../DefinitionPopoverContents.tsx | 45 +++--- .../definitionPopoverLogic.ts | 6 + .../lib/components/DefinitionPopover/utils.ts | 1 + .../components/PropertyFilters/utils.test.ts | 4 +- .../lib/components/PropertyFilters/utils.ts | 16 +- .../TaxonomicFilter/InfiniteList.tsx | 2 + .../taxonomicFilterLogic.test.ts | 22 +-- .../TaxonomicFilter/taxonomicFilterLogic.tsx | 32 ++-- .../lib/components/TaxonomicFilter/types.ts | 2 +- frontend/src/lib/constants.tsx | 1 + frontend/src/lib/taxonomy.test.tsx | 4 +- frontend/src/lib/taxonomy.tsx | 19 +-- frontend/src/lib/utils.tsx | 2 + .../src/models/propertyDefinitionsModel.ts | 83 +++++++---- .../nodes/InsightViz/GlobalAndOrFilters.tsx | 2 +- .../queries/nodes/InsightViz/TrendsSeries.tsx | 2 +- frontend/src/queries/schema.json | 3 + frontend/src/queries/schema.ts | 3 +- .../ActionFilterRow/ActionFilterRow.tsx | 2 +- .../TaxonomicBreakdownPopover.tsx | 2 +- .../web-analytics/WebPropertyFilters.tsx | 48 ++---- frontend/src/test/mocks.ts | 11 +- frontend/src/types.ts | 1 + mypy-baseline.txt | 9 +- posthog/api/__init__.py | 2 + posthog/api/property_definition.py | 12 +- posthog/api/session.py | 56 +++++++ .../api/test/__snapshots__/test_api_docs.ambr | 2 + posthog/api/test/test_session.py | 137 ++++++++++++++++++ posthog/hogql/database/schema/channel_type.py | 20 +++ posthog/hogql/database/schema/sessions.py | 113 ++++++++++++++- .../database/schema/test/test_sessions.py | 33 +++++ .../web_analytics_query_runner.py | 5 +- posthog/models/property/util.py | 2 +- posthog/models/sessions/sql.py | 41 ++++++ posthog/schema.py | 8 +- 38 files changed, 641 insertions(+), 136 deletions(-) create mode 100644 posthog/api/session.py create mode 100644 posthog/api/test/test_session.py diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 5e3a3aab0604f..09e3924e21deb 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -388,6 +388,10 @@ class ApiRequest { .withQueryString(queryParams) } + public sessionPropertyDefinitions(teamId?: TeamType['id']): ApiRequest { + return this.projectsDetail(teamId).addPathComponent('sessions').addPathComponent('property_definitions') + } + public dataManagementActivity(teamId?: TeamType['id']): ApiRequest { return this.projectsDetail(teamId).addPathComponent('data_management').addPathComponent('activity') } @@ -1212,6 +1216,23 @@ const api = { }, }, + sessions: { + async propertyDefinitions({ + teamId = ApiConfig.getCurrentTeamId(), + search, + properties, + }: { + teamId?: TeamType['id'] + search?: string + properties?: string[] + }): Promise> { + return new ApiRequest() + .sessionPropertyDefinitions(teamId) + .withQueryString(toParams({ search, ...(properties ? { properties: properties.join(',') } : {}) })) + .get() + }, + }, + cohorts: { async get(cohortId: CohortType['id']): Promise { return await new ApiRequest().cohortsDetail(cohortId).get() diff --git a/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx b/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx index 4d93fd38f2ae9..807f1097e45c2 100644 --- a/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx +++ b/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx @@ -124,7 +124,8 @@ function Example({ value }: { value?: string }): JSX.Element { type === TaxonomicFilterGroupType.EventFeatureFlags || type === TaxonomicFilterGroupType.PersonProperties || type === TaxonomicFilterGroupType.GroupsPrefix || - type === TaxonomicFilterGroupType.Metadata + type === TaxonomicFilterGroupType.Metadata || + type === TaxonomicFilterGroupType.SessionProperties ) { data = getCoreFilterDefinition(value, type) } else if (type === TaxonomicFilterGroupType.Elements) { diff --git a/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx b/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx index d6419882b3327..f578e4d37b07f 100644 --- a/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx +++ b/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx @@ -78,6 +78,7 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element isCohort, isDataWarehouse, isProperty, + hasSentAs, } = useValues(definitionPopoverLogic) const { setLocalDefinition } = useActions(definitionPopoverLogic) @@ -142,13 +143,17 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element /> - - - {_definition.name}} - /> - + {hasSentAs ? ( + <> + + + {_definition.name}} + /> + + + ) : null} ) } @@ -176,17 +181,21 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element - - - - {_definition.name !== '' ? _definition.name : (empty string)} - - } - /> - + {hasSentAs ? ( + <> + + + + {_definition.name !== '' ? _definition.name : (empty string)} + + } + /> + + + ) : null} ) } diff --git a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts index c3baac7ce76ca..38d9af8eb5311 100644 --- a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts +++ b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts @@ -176,11 +176,17 @@ export const definitionPopoverLogic = kea([ [ TaxonomicFilterGroupType.PersonProperties, TaxonomicFilterGroupType.EventProperties, + TaxonomicFilterGroupType.SessionProperties, TaxonomicFilterGroupType.EventFeatureFlags, TaxonomicFilterGroupType.NumericalEventProperties, TaxonomicFilterGroupType.Metadata, ].includes(type) || type.startsWith(TaxonomicFilterGroupType.GroupsPrefix), ], + hasSentAs: [ + (s) => [s.type, s.isProperty, s.isEvent], + (type, isProperty, isEvent) => + isEvent || (isProperty && type !== TaxonomicFilterGroupType.SessionProperties), + ], isCohort: [(s) => [s.type], (type) => type === TaxonomicFilterGroupType.Cohorts], isDataWarehouse: [(s) => [s.type], (type) => type === TaxonomicFilterGroupType.DataWarehouse], viewFullDetailUrl: [ diff --git a/frontend/src/lib/components/DefinitionPopover/utils.ts b/frontend/src/lib/components/DefinitionPopover/utils.ts index f828aea981fc2..a12c76237da42 100644 --- a/frontend/src/lib/components/DefinitionPopover/utils.ts +++ b/frontend/src/lib/components/DefinitionPopover/utils.ts @@ -48,6 +48,7 @@ export function getSingularType(type: TaxonomicFilterGroupType): string { case TaxonomicFilterGroupType.EventProperties: case TaxonomicFilterGroupType.PersonProperties: case TaxonomicFilterGroupType.GroupsPrefix: // Group properties + case TaxonomicFilterGroupType.SessionProperties: return 'property' case TaxonomicFilterGroupType.EventFeatureFlags: return 'feature' diff --git a/frontend/src/lib/components/PropertyFilters/utils.test.ts b/frontend/src/lib/components/PropertyFilters/utils.test.ts index 33ad74f8e35d6..da3f32d7e1b3c 100644 --- a/frontend/src/lib/components/PropertyFilters/utils.test.ts +++ b/frontend/src/lib/components/PropertyFilters/utils.test.ts @@ -83,7 +83,7 @@ describe('propertyFilterTypeToTaxonomicFilterType()', () => { ...baseFilter, type: PropertyFilterType.Session, } as SessionPropertyFilter) - ).toEqual(TaxonomicFilterGroupType.Sessions) + ).toEqual(TaxonomicFilterGroupType.SessionProperties) expect(propertyFilterTypeToTaxonomicFilterType({ ...baseFilter, type: PropertyFilterType.HogQL })).toEqual( TaxonomicFilterGroupType.HogQLExpression ) @@ -122,7 +122,7 @@ describe('breakdownFilterToTaxonomicFilterType()', () => { TaxonomicFilterGroupType.EventProperties ) expect(breakdownFilterToTaxonomicFilterType({ ...baseFilter, breakdown_type: 'session' })).toEqual( - TaxonomicFilterGroupType.Sessions + TaxonomicFilterGroupType.SessionProperties ) expect(breakdownFilterToTaxonomicFilterType({ ...baseFilter, breakdown_type: 'hogql' })).toEqual( TaxonomicFilterGroupType.HogQLExpression diff --git a/frontend/src/lib/components/PropertyFilters/utils.ts b/frontend/src/lib/components/PropertyFilters/utils.ts index ad135c0525b0e..63833cbd7acb5 100644 --- a/frontend/src/lib/components/PropertyFilters/utils.ts +++ b/frontend/src/lib/components/PropertyFilters/utils.ts @@ -97,7 +97,7 @@ export const PROPERTY_FILTER_TYPE_TO_TAXONOMIC_FILTER_GROUP_TYPE: Omit< [PropertyFilterType.Feature]: TaxonomicFilterGroupType.EventFeatureFlags, [PropertyFilterType.Cohort]: TaxonomicFilterGroupType.Cohorts, [PropertyFilterType.Element]: TaxonomicFilterGroupType.Elements, - [PropertyFilterType.Session]: TaxonomicFilterGroupType.Sessions, + [PropertyFilterType.Session]: TaxonomicFilterGroupType.SessionProperties, [PropertyFilterType.HogQL]: TaxonomicFilterGroupType.HogQLExpression, [PropertyFilterType.Group]: TaxonomicFilterGroupType.GroupsPrefix, [PropertyFilterType.DataWarehouse]: TaxonomicFilterGroupType.DataWarehouse, @@ -183,10 +183,14 @@ export function isEventPropertyFilter(filter?: AnyFilterLike | null): filter is export function isPersonPropertyFilter(filter?: AnyFilterLike | null): filter is PersonPropertyFilter { return filter?.type === PropertyFilterType.Person } -export function isEventPropertyOrPersonPropertyFilter( +export function isEventPersonOrSessionPropertyFilter( filter?: AnyFilterLike | null -): filter is EventPropertyFilter | PersonPropertyFilter { - return filter?.type === PropertyFilterType.Event || filter?.type === PropertyFilterType.Person +): filter is EventPropertyFilter | PersonPropertyFilter | SessionPropertyFilter { + return ( + filter?.type === PropertyFilterType.Event || + filter?.type === PropertyFilterType.Person || + filter?.type === PropertyFilterType.Session + ) } export function isElementPropertyFilter(filter?: AnyFilterLike | null): filter is ElementPropertyFilter { return filter?.type === PropertyFilterType.Element @@ -264,7 +268,7 @@ const propertyFilterMapping: Partial
@@ -160,6 +161,7 @@ const selectedItemHasPopover = ( TaxonomicFilterGroupType.Cohorts, TaxonomicFilterGroupType.CohortsWithAllUsers, TaxonomicFilterGroupType.Metadata, + TaxonomicFilterGroupType.SessionProperties, ].includes(listGroupType) || listGroupType.startsWith(TaxonomicFilterGroupType.GroupsPrefix)) ) diff --git a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.test.ts b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.test.ts index 2c6f0ff84c2db..0dbbe75ddd38d 100644 --- a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.test.ts +++ b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.test.ts @@ -45,7 +45,7 @@ describe('taxonomicFilterLogic', () => { TaxonomicFilterGroupType.Events, TaxonomicFilterGroupType.Actions, TaxonomicFilterGroupType.Elements, - TaxonomicFilterGroupType.Sessions, + TaxonomicFilterGroupType.SessionProperties, ], } logic = taxonomicFilterLogic(logicProps) @@ -62,7 +62,7 @@ describe('taxonomicFilterLogic', () => { infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Events }), infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Actions }), infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Elements }), - infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Sessions }), + infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.SessionProperties }), ]) expect( infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Cohorts }).isMounted() @@ -76,7 +76,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 1, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 4, - [TaxonomicFilterGroupType.Sessions]: 1, + [TaxonomicFilterGroupType.SessionProperties]: 1, }, }) .toDispatchActions(['infiniteListResultsReceived']) @@ -87,7 +87,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 157, [TaxonomicFilterGroupType.Actions]: 0, // not mocked [TaxonomicFilterGroupType.Elements]: 4, - [TaxonomicFilterGroupType.Sessions]: 1, + [TaxonomicFilterGroupType.SessionProperties]: 1, }, }) }) @@ -110,7 +110,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 4, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 0, - [TaxonomicFilterGroupType.Sessions]: 0, + [TaxonomicFilterGroupType.SessionProperties]: 0, }, }) @@ -127,7 +127,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 0, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 1, - [TaxonomicFilterGroupType.Sessions]: 0, + [TaxonomicFilterGroupType.SessionProperties]: 0, }, }) @@ -144,7 +144,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 0, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 0, - [TaxonomicFilterGroupType.Sessions]: 0, + [TaxonomicFilterGroupType.SessionProperties]: 0, }, }) @@ -161,13 +161,13 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 157, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 4, - [TaxonomicFilterGroupType.Sessions]: 1, + [TaxonomicFilterGroupType.SessionProperties]: 1, }, }) // move right, skipping Actions await expectLogic(logic, () => logic.actions.tabRight()).toMatchValues({ - activeTab: TaxonomicFilterGroupType.Sessions, + activeTab: TaxonomicFilterGroupType.SessionProperties, }) await expectLogic(logic, () => logic.actions.tabRight()).toMatchValues({ activeTab: TaxonomicFilterGroupType.Events, @@ -181,7 +181,7 @@ describe('taxonomicFilterLogic', () => { activeTab: TaxonomicFilterGroupType.Events, }) await expectLogic(logic, () => logic.actions.tabLeft()).toMatchValues({ - activeTab: TaxonomicFilterGroupType.Sessions, + activeTab: TaxonomicFilterGroupType.SessionProperties, }) await expectLogic(logic, () => logic.actions.tabLeft()).toMatchValues({ activeTab: TaxonomicFilterGroupType.Elements, @@ -201,7 +201,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 4, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 0, - [TaxonomicFilterGroupType.Sessions]: 0, + [TaxonomicFilterGroupType.SessionProperties]: 0, }, }) }) diff --git a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx index 966b33c5527ed..f35dbda2f7cf5 100644 --- a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx +++ b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx @@ -11,7 +11,9 @@ import { TaxonomicFilterLogicProps, TaxonomicFilterValue, } from 'lib/components/TaxonomicFilter/types' +import { FEATURE_FLAGS } from 'lib/constants' import { IconCohort } from 'lib/lemon-ui/icons' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { CORE_FILTER_DEFINITIONS_BY_GROUP } from 'lib/taxonomy' import { capitalizeFirstLetter, pluralize, toParams } from 'lib/utils' import { getEventDefinitionIcon, getPropertyDefinitionIcon } from 'scenes/data-management/events/DefinitionHeader' @@ -168,6 +170,7 @@ export const taxonomicFilterLogic = kea([ s.metadataSource, s.excludedProperties, s.propertyAllowList, + featureFlagLogic.selectors.featureFlags, ], ( teamId, @@ -177,7 +180,8 @@ export const taxonomicFilterLogic = kea([ schemaColumns, metadataSource, excludedProperties, - propertyAllowList + propertyAllowList, + featureFlags ): TaxonomicFilterGroup[] => { const groups: TaxonomicFilterGroup[] = [ { @@ -486,18 +490,26 @@ export const taxonomicFilterLogic = kea([ getPopoverHeader: () => 'Notebooks', }, { - name: 'Sessions', + name: 'Session Properties', searchPlaceholder: 'sessions', - type: TaxonomicFilterGroupType.Sessions, - options: [ - { - name: 'Session duration', - value: '$session_duration', - }, - ], + type: TaxonomicFilterGroupType.SessionProperties, + options: featureFlags[FEATURE_FLAGS.SESSION_TABLE_PROPERTY_FILTERS] + ? undefined + : [ + { + id: '$session_duration', + name: '$session_duration', + property_type: 'Duration', + is_numerical: true, + }, + ], getName: (option: any) => option.name, - getValue: (option: any) => option.value, + getValue: (option) => option.name, getPopoverHeader: () => 'Session', + endpoint: featureFlags[FEATURE_FLAGS.SESSION_TABLE_PROPERTY_FILTERS] + ? `api/projects/${teamId}/sessions/property_definitions` + : undefined, + getIcon: getPropertyDefinitionIcon, }, { name: 'HogQL', diff --git a/frontend/src/lib/components/TaxonomicFilter/types.ts b/frontend/src/lib/components/TaxonomicFilter/types.ts index 8b46784a19b7e..37f1c95b45daa 100644 --- a/frontend/src/lib/components/TaxonomicFilter/types.ts +++ b/frontend/src/lib/components/TaxonomicFilter/types.ts @@ -105,7 +105,7 @@ export enum TaxonomicFilterGroupType { Plugins = 'plugins', Dashboards = 'dashboards', GroupNamesPrefix = 'name_groups', - Sessions = 'sessions', + SessionProperties = 'session_properties', HogQLExpression = 'hogql_expression', Notebooks = 'notebooks', } diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 0db68836f76d5..86450c67f19e0 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -212,6 +212,7 @@ export const FEATURE_FLAGS = { EMAIL_VERIFICATION_TICKET_SUBMISSION: 'email-verification-ticket-submission', // owner: #team-growth TOOLBAR_HEATMAPS: 'toolbar-heatmaps', // owner: #team-replay THEME: 'theme', // owner: @aprilfools + SESSION_TABLE_PROPERTY_FILTERS: 'session-table-property-filters', // owner: @robbie-c } as const export type FeatureFlagKey = (typeof FEATURE_FLAGS)[keyof typeof FEATURE_FLAGS] diff --git a/frontend/src/lib/taxonomy.test.tsx b/frontend/src/lib/taxonomy.test.tsx index 3805f9d7670ad..e8820fbad0c32 100644 --- a/frontend/src/lib/taxonomy.test.tsx +++ b/frontend/src/lib/taxonomy.test.tsx @@ -26,10 +26,10 @@ describe('taxonomy', () => { }) describe('session properties', () => { - const sessionPropertyNames = Object.keys(CORE_FILTER_DEFINITIONS_BY_GROUP.sessions) + const sessionPropertyNames = Object.keys(CORE_FILTER_DEFINITIONS_BY_GROUP.session_properties) it('should have an $initial_referring_domain property', () => { const property: CoreFilterDefinition = - CORE_FILTER_DEFINITIONS_BY_GROUP.sessions['$initial_referring_domain'] + CORE_FILTER_DEFINITIONS_BY_GROUP.session_properties['$initial_referring_domain'] expect(property.label).toEqual('Initial Referring Domain') }) it(`should have every property in SESSION_PROPERTIES_ADAPTED_FROM_PERSON`, () => { diff --git a/frontend/src/lib/taxonomy.tsx b/frontend/src/lib/taxonomy.tsx index 9b5297cf62955..d52ee74c583c0 100644 --- a/frontend/src/lib/taxonomy.tsx +++ b/frontend/src/lib/taxonomy.tsx @@ -993,7 +993,7 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { }, numerical_event_properties: {}, // Same as event properties, see assignment below person_properties: {}, // Currently person properties are the same as event properties, see assignment below - sessions: { + session_properties: { $session_duration: { label: 'Session duration', description: ( @@ -1006,13 +1006,13 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { ), examples: ['01:04:12'], }, - $min_timestamp: { - label: 'First timestamp', + $start_timestamp: { + label: 'Start timestamp', description: The timestamp of the first event from this session., examples: [new Date().toISOString()], }, - $max_timestamp: { - label: 'Last timestamp', + $end_timestamp: { + label: 'End timestamp', description: The timestamp of the last event from this session, examples: [new Date().toISOString()], }, @@ -1022,7 +1022,7 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { examples: ['https://example.com/interesting-article?parameter=true'], }, $exit_url: { - label: 'Entry URL', + label: 'Exit URL', description: The last URL visited in this session, examples: ['https://example.com/interesting-article?parameter=true'], }, @@ -1036,7 +1036,7 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { description: The number of autocapture events in this session, examples: ['123'], }, - $initial_channel_type: { + $channel_type: { label: 'Channel type', description: What type of acquisition channel this traffic came from., examples: ['Paid Search', 'Organic Video', 'Direct'], @@ -1079,20 +1079,21 @@ for (const [key, value] of Object.entries(CORE_FILTER_DEFINITIONS_BY_GROUP.event CORE_FILTER_DEFINITIONS_BY_GROUP.person_properties[key] = value } if (SESSION_INITIAL_PROPERTIES_ADAPTED_FROM_EVENTS.has(key)) { - CORE_FILTER_DEFINITIONS_BY_GROUP.sessions[`$initial_${key.replace(/^\$/, '')}`] = { + CORE_FILTER_DEFINITIONS_BY_GROUP.session_properties[`$initial_${key.replace(/^\$/, '')}`] = { ...value, label: `Initial ${value.label}`, description: 'description' in value ? `${value.description} Data from the first event in this session.` : 'Data from the first event in this session.', + examples: 'examples' in value ? value.examples : undefined, } } } // We treat `$session_duration` as an event property in the context of series `math`, but it's fake in a sense CORE_FILTER_DEFINITIONS_BY_GROUP.event_properties.$session_duration = - CORE_FILTER_DEFINITIONS_BY_GROUP.sessions.$session_duration + CORE_FILTER_DEFINITIONS_BY_GROUP.session_properties.$session_duration export const PROPERTY_KEYS = Object.keys(CORE_FILTER_DEFINITIONS_BY_GROUP.event_properties) diff --git a/frontend/src/lib/utils.tsx b/frontend/src/lib/utils.tsx index ab32f34b314f3..20b8114375939 100644 --- a/frontend/src/lib/utils.tsx +++ b/frontend/src/lib/utils.tsx @@ -1597,6 +1597,8 @@ export function promiseResolveReject(): { return { resolve: resolve!, reject: reject!, promise } } +export type AsyncReturnType any> = T extends (...args: any) => Promise ? R : any + export function calculateDays(timeValue: number, timeUnit: TimeUnitType): number { if (timeUnit === TimeUnitType.Year) { return timeValue * 365 diff --git a/frontend/src/models/propertyDefinitionsModel.ts b/frontend/src/models/propertyDefinitionsModel.ts index 338e60a5e956f..0bc2dfae5c11b 100644 --- a/frontend/src/models/propertyDefinitionsModel.ts +++ b/frontend/src/models/propertyDefinitionsModel.ts @@ -1,5 +1,5 @@ -import { actions, kea, listeners, path, reducers, selectors } from 'kea' -import api, { ApiMethodOptions } from 'lib/api' +import { actions, connect, kea, listeners, path, reducers, selectors } from 'kea' +import api, { ApiMethodOptions, CountedPaginatedResponse } from 'lib/api' import { TaxonomicFilterValue } from 'lib/components/TaxonomicFilter/types' import { dayjs } from 'lib/dayjs' import { captureTimeToSeeData } from 'lib/internalMetrics' @@ -95,8 +95,42 @@ const checkOrLoadPropertyDefinition = ( return null } +const getEndpoint = ( + teamId: number, + type: PropertyDefinitionType, + propertyKey: string, + eventNames: string[] | undefined, + newInput: string | undefined +): string => { + let eventParams = '' + for (const eventName of eventNames || []) { + eventParams += `&event_name=${eventName}` + } + + if (type === PropertyDefinitionType.Session) { + return ( + `api/projects/${teamId}/${type}s/values/?key=` + + encodeURIComponent(propertyKey) + + (newInput ? '&value=' + encodeURIComponent(newInput) : '') + + eventParams + ) + } + + return ( + 'api/' + + type + + '/values/?key=' + + encodeURIComponent(propertyKey) + + (newInput ? '&value=' + encodeURIComponent(newInput) : '') + + eventParams + ) +} + export const propertyDefinitionsModel = kea([ path(['models', 'propertyDefinitionsModel']), + connect({ + values: [teamLogic, ['currentTeamId']], + }), actions({ // public loadPropertyDefinitions: ( @@ -125,10 +159,12 @@ export const propertyDefinitionsModel = kea([ propertyDefinitionStorage: [ { ...localProperties } as PropertyDefinitionStorage, { - updatePropertyDefinitions: (state, { propertyDefinitions }) => ({ - ...state, - ...propertyDefinitions, - }), + updatePropertyDefinitions: (state, { propertyDefinitions }) => { + return { + ...state, + ...propertyDefinitions, + } + }, }, ], options: [ @@ -179,7 +215,7 @@ export const propertyDefinitionsModel = kea([ // take the first 50 pending properties to avoid the 4k query param length limit const allPending = values.pendingProperties.slice(0, 50) const pendingByType: Record< - 'event' | 'person' | 'group/0' | 'group/1' | 'group/2' | 'group/3' | 'group/4', + 'event' | 'person' | 'group/0' | 'group/1' | 'group/2' | 'group/3' | 'group/4' | 'session', string[] > = { event: [], @@ -189,6 +225,7 @@ export const propertyDefinitionsModel = kea([ 'group/2': [], 'group/3': [], 'group/4': [], + session: [], } for (const key of allPending) { let [type, ...rest] = key.split('/') @@ -226,10 +263,17 @@ export const propertyDefinitionsModel = kea([ } // and then fetch them - const propertyDefinitions = await api.propertyDefinitions.list({ - properties: pending, - ...queryParams, - }) + let propertyDefinitions: CountedPaginatedResponse + if (type === 'session') { + propertyDefinitions = await api.sessions.propertyDefinitions({ + properties: pending, + }) + } else { + propertyDefinitions = await api.propertyDefinitions.list({ + properties: pending, + ...queryParams, + }) + } for (const propertyDefinition of propertyDefinitions.results) { newProperties[`${type}/${propertyDefinition.name}`] = propertyDefinition @@ -268,10 +312,10 @@ export const propertyDefinitionsModel = kea([ }, loadPropertyValues: async ({ endpoint, type, newInput, propertyKey, eventNames }, breakpoint) => { - if (['cohort', 'session'].includes(type)) { + if (['cohort'].includes(type)) { return } - if (!propertyKey) { + if (!propertyKey || values.currentTeamId === null) { return } @@ -286,19 +330,8 @@ export const propertyDefinitionsModel = kea([ signal: cache.abortController.signal, } - let eventParams = '' - for (const eventName of eventNames || []) { - eventParams += `&event_name=${eventName}` - } - const propValues: PropValue[] = await api.get( - endpoint || - 'api/' + - type + - '/values/?key=' + - encodeURIComponent(propertyKey) + - (newInput ? '&value=' + encodeURIComponent(newInput) : '') + - eventParams, + endpoint || getEndpoint(values.currentTeamId, type, propertyKey, eventNames, newInput), methodOptions ) breakpoint() diff --git a/frontend/src/queries/nodes/InsightViz/GlobalAndOrFilters.tsx b/frontend/src/queries/nodes/InsightViz/GlobalAndOrFilters.tsx index 8485b9a71ee84..351e82136133a 100644 --- a/frontend/src/queries/nodes/InsightViz/GlobalAndOrFilters.tsx +++ b/frontend/src/queries/nodes/InsightViz/GlobalAndOrFilters.tsx @@ -27,7 +27,7 @@ export function GlobalAndOrFilters({ insightProps }: EditorFilterProps): JSX.Ele ...groupsTaxonomicTypes, TaxonomicFilterGroupType.Cohorts, TaxonomicFilterGroupType.Elements, - ...(isTrends ? [TaxonomicFilterGroupType.Sessions] : []), + ...(isTrends ? [TaxonomicFilterGroupType.SessionProperties] : []), TaxonomicFilterGroupType.HogQLExpression, ...(featureFlags[FEATURE_FLAGS.DATA_WAREHOUSE] && featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS] ? [TaxonomicFilterGroupType.DataWarehousePersonProperties] diff --git a/frontend/src/queries/nodes/InsightViz/TrendsSeries.tsx b/frontend/src/queries/nodes/InsightViz/TrendsSeries.tsx index 38b96c2162aca..dd75cd96a6f97 100644 --- a/frontend/src/queries/nodes/InsightViz/TrendsSeries.tsx +++ b/frontend/src/queries/nodes/InsightViz/TrendsSeries.tsx @@ -34,7 +34,7 @@ export function TrendsSeries(): JSX.Element | null { ...groupsTaxonomicTypes, TaxonomicFilterGroupType.Cohorts, TaxonomicFilterGroupType.Elements, - ...(isTrends ? [TaxonomicFilterGroupType.Sessions] : []), + ...(isTrends ? [TaxonomicFilterGroupType.SessionProperties] : []), TaxonomicFilterGroupType.HogQLExpression, TaxonomicFilterGroupType.DataWarehouseProperties, ...(featureFlags[FEATURE_FLAGS.DATA_WAREHOUSE] && featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS] diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 868156528ece8..1ef498660d74a 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -5667,6 +5667,9 @@ }, { "$ref": "#/definitions/PersonPropertyFilter" + }, + { + "$ref": "#/definitions/SessionPropertyFilter" } ] }, diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 01708078b175b..a923dfaec77cf 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -23,6 +23,7 @@ import { PropertyGroupFilter, PropertyMathType, RetentionFilterType, + SessionPropertyFilter, StickinessFilterType, TrendsFilterType, } from '~/types' @@ -985,7 +986,7 @@ export interface SessionsTimelineQuery extends DataNode { before?: string response?: SessionsTimelineQueryResponse } -export type WebAnalyticsPropertyFilter = EventPropertyFilter | PersonPropertyFilter +export type WebAnalyticsPropertyFilter = EventPropertyFilter | PersonPropertyFilter | SessionPropertyFilter export type WebAnalyticsPropertyFilters = WebAnalyticsPropertyFilter[] export interface WebAnalyticsQueryBase { diff --git a/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx b/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx index 0cb3eaeb086b3..f544e601e506d 100644 --- a/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx +++ b/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx @@ -390,7 +390,7 @@ export function ActionFilterRow({ groupTypes={[ TaxonomicFilterGroupType.DataWarehouseProperties, TaxonomicFilterGroupType.NumericalEventProperties, - TaxonomicFilterGroupType.Sessions, + TaxonomicFilterGroupType.SessionProperties, ]} schemaColumns={ filter.type == TaxonomicFilterGroupType.DataWarehouse && filter.name diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownPopover.tsx b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownPopover.tsx index abbcb7020c116..9a8df4a8c392f 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownPopover.tsx +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownPopover.tsx @@ -28,7 +28,7 @@ export const TaxonomicBreakdownPopover = ({ open, setOpen, children }: Taxonomic TaxonomicFilterGroupType.EventFeatureFlags, ...groupsTaxonomicTypes, TaxonomicFilterGroupType.CohortsWithAllUsers, - ...(includeSessions ? [TaxonomicFilterGroupType.Sessions] : []), + ...(includeSessions ? [TaxonomicFilterGroupType.SessionProperties] : []), TaxonomicFilterGroupType.HogQLExpression, TaxonomicFilterGroupType.DataWarehouseProperties, ] diff --git a/frontend/src/scenes/web-analytics/WebPropertyFilters.tsx b/frontend/src/scenes/web-analytics/WebPropertyFilters.tsx index 0f65abec6f0c9..5237a78e3cb8b 100644 --- a/frontend/src/scenes/web-analytics/WebPropertyFilters.tsx +++ b/frontend/src/scenes/web-analytics/WebPropertyFilters.tsx @@ -1,6 +1,9 @@ +import { useValues } from 'kea' import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters' -import { isEventPropertyOrPersonPropertyFilter } from 'lib/components/PropertyFilters/utils' +import { isEventPersonOrSessionPropertyFilter } from 'lib/components/PropertyFilters/utils' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { WebAnalyticsPropertyFilters } from '~/queries/schema' @@ -11,42 +14,23 @@ export const WebPropertyFilters = ({ webAnalyticsFilters: WebAnalyticsPropertyFilters setWebAnalyticsFilters: (filters: WebAnalyticsPropertyFilters) => void }): JSX.Element => { + const { featureFlags } = useValues(featureFlagLogic) + return ( setWebAnalyticsFilters(filters.filter(isEventPropertyOrPersonPropertyFilter))} + taxonomicGroupTypes={ + featureFlags[FEATURE_FLAGS.SESSION_TABLE_PROPERTY_FILTERS] + ? [ + TaxonomicFilterGroupType.SessionProperties, + TaxonomicFilterGroupType.EventProperties, + TaxonomicFilterGroupType.PersonProperties, + ] + : [TaxonomicFilterGroupType.EventProperties, TaxonomicFilterGroupType.PersonProperties] + } + onChange={(filters) => setWebAnalyticsFilters(filters.filter(isEventPersonOrSessionPropertyFilter))} propertyFilters={webAnalyticsFilters} pageKey="web-analytics" eventNames={['$pageview', '$pageleave', '$autocapture']} - propertyAllowList={{ - [TaxonomicFilterGroupType.EventProperties]: [ - '$pathname', - '$host', - '$browser', - '$os', - '$device_type', - '$geoip_country_code', - '$geoip_subdivision_1_code', - '$geoip_city_name', - // re-enable after https://github.com/PostHog/posthog-js/pull/875 is merged - // '$client_session_initial_pathname', - // '$client_session_initial_referring_host', - // '$client_session_initial_utm_source', - // '$client_session_initial_utm_campaign', - // '$client_session_initial_utm_medium', - // '$client_session_initial_utm_content', - // '$client_session_initial_utm_term', - ], - [TaxonomicFilterGroupType.PersonProperties]: [ - '$initial_pathname', - '$initial_referring_domain', - '$initial_utm_source', - '$initial_utm_campaign', - '$initial_utm_medium', - '$initial_utm_content', - '$initial_utm_term', - ], - }} /> ) } diff --git a/frontend/src/test/mocks.ts b/frontend/src/test/mocks.ts index dcd926d4f1e7a..78cba1619bfb8 100644 --- a/frontend/src/test/mocks.ts +++ b/frontend/src/test/mocks.ts @@ -51,7 +51,7 @@ export const mockEventDefinitions: EventDefinition[] = [ 'test event', '$click', '$autocapture', - 'search', + 'search term', 'other event', ...Array(150), ].map((name, index) => ({ @@ -89,6 +89,15 @@ export const mockEventPropertyDefinitions: PropertyDefinition[] = [ is_seen_on_filtered_events: (name || '').includes('$'), })) +export const mockSessionPropertyDefinitions: PropertyDefinition[] = ['$session_duration', '$initial_utm_source'].map( + (name) => ({ + ...mockEventPropertyDefinition, + id: name, + name: name, + description: `${name} is the best!`, + }) +) + export const mockPersonProperty = { name: '$browser_version', count: 1, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index cfdc09496ef93..955735d6b5bd3 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2815,6 +2815,7 @@ export enum PropertyDefinitionType { Event = 'event', Person = 'person', Group = 'group', + Session = 'session', } export interface PropertyDefinition { diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 9b607b6222cd3..b1e56d812c2f0 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -66,11 +66,8 @@ posthog/hogql/database/schema/person_distinct_ids.py:0: error: Argument 1 to "se posthog/hogql/database/schema/person_distinct_id_overrides.py:0: error: Argument 1 to "select_from_person_distinct_id_overrides_table" has incompatible type "dict[str, list[str]]"; expected "dict[str, list[str | int]]" [arg-type] posthog/plugins/utils.py:0: error: Subclass of "str" and "bytes" cannot exist: would have incompatible method signatures [unreachable] posthog/plugins/utils.py:0: error: Statement is unreachable [unreachable] +posthog/clickhouse/kafka_engine.py:0: error: Argument 1 to "join" of "str" has incompatible type "list"; expected "Iterable[str]" [arg-type] posthog/models/filters/base_filter.py:0: error: "HogQLContext" has no attribute "person_on_events_mode" [attr-defined] -posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined] -posthog/hogql/database/database.py:0: error: Incompatible types (expression has type "Literal['view', 'lazy_table']", TypedDict item "type" has type "Literal['integer', 'float', 'string', 'datetime', 'date', 'boolean', 'array', 'json', 'lazy_table', 'virtual_table', 'field_traverser', 'expression']") [typeddict-item] -posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Argument 1 to "create_hogql_database" has incompatible type "int | None"; expected "int" [arg-type] -posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment] posthog/models/user.py:0: error: Incompatible types in assignment (expression has type "None", base class "AbstractUser" defined the type as "CharField[str | int | Combinable, str]") [assignment] posthog/models/user.py:0: error: Incompatible types in assignment (expression has type "posthog.models.user.UserManager", base class "AbstractUser" defined the type as "django.contrib.auth.models.UserManager[AbstractUser]") [assignment] posthog/models/user.py:0: error: Cannot override writeable attribute with read-only property [override] @@ -82,6 +79,10 @@ posthog/models/user.py:0: note: bool posthog/models/user.py:0: error: "User" has no attribute "social_auth" [attr-defined] posthog/models/user.py:0: error: "User" has no attribute "social_auth" [attr-defined] posthog/models/person/person.py:0: error: Incompatible types in assignment (expression has type "list[Never]", variable has type "ValuesQuerySet[PersonDistinctId, str]") [assignment] +posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined] +posthog/hogql/database/database.py:0: error: Incompatible types (expression has type "Literal['view', 'lazy_table']", TypedDict item "type" has type "Literal['integer', 'float', 'string', 'datetime', 'date', 'boolean', 'array', 'json', 'lazy_table', 'virtual_table', 'field_traverser', 'expression']") [typeddict-item] +posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Argument 1 to "create_hogql_database" has incompatible type "int | None"; expected "int" [arg-type] +posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment] posthog/models/feature_flag/flag_matching.py:0: error: Statement is unreachable [unreachable] posthog/hogql_queries/utils/query_date_range.py:0: error: Incompatible return value type (got "str", expected "Literal['hour', 'day', 'week', 'month']") [return-value] posthog/hogql_queries/utils/query_date_range.py:0: error: Item "None" of "dict[str, int] | None" has no attribute "get" [union-attr] diff --git a/posthog/api/__init__.py b/posthog/api/__init__.py index e9828f74e7f1d..453a2c4417d82 100644 --- a/posthog/api/__init__.py +++ b/posthog/api/__init__.py @@ -5,6 +5,7 @@ from posthog.settings import EE_AVAILABLE from posthog.warehouse.api import external_data_source, saved_query, table, view_link, external_data_schema from ..heatmaps.heatmaps_api import LegacyHeatmapViewSet, HeatmapViewSet +from .session import SessionViewSet from ..session_recordings.session_recording_api import SessionRecordingViewSet from . import ( activity_log, @@ -333,6 +334,7 @@ def api_not_found(request): ["team_id"], ) projects_router.register(r"heatmaps", HeatmapViewSet, "project_heatmaps", ["team_id"]) +projects_router.register(r"sessions", SessionViewSet, "project_sessions", ["team_id"]) if EE_AVAILABLE: from ee.clickhouse.views.experiments import ClickhouseExperimentsViewSet diff --git a/posthog/api/property_definition.py b/posthog/api/property_definition.py index 6a87fc6f348d7..7db63497d5a9d 100644 --- a/posthog/api/property_definition.py +++ b/posthog/api/property_definition.py @@ -35,7 +35,7 @@ class PropertyDefinitionQuerySerializer(serializers.Serializer): ) type = serializers.ChoiceField( - choices=["event", "person", "group"], + choices=["event", "person", "group", "session"], help_text="What property definitions to return", default="event", ) @@ -192,6 +192,16 @@ def with_type_filter(self, type: str, group_type_index: Optional[int]): "group_type_index": group_type_index, }, ) + elif type == "session": + return dataclasses.replace( + self, + should_join_event_property=False, + params={ + **self.params, + "type": PropertyDefinition.Type.SESSION, + "group_type_index": -1, + }, + ) def with_event_property_filter( self, event_names: Optional[str], filter_by_event_names: Optional[bool] diff --git a/posthog/api/session.py b/posthog/api/session.py new file mode 100644 index 0000000000000..b4c79600d1999 --- /dev/null +++ b/posthog/api/session.py @@ -0,0 +1,56 @@ +import json + +from rest_framework import request, response, viewsets +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError + +from posthog.api.routing import TeamAndOrgViewSetMixin +from posthog.hogql.database.schema.sessions import get_lazy_session_table_properties, get_lazy_session_table_values +from posthog.rate_limit import ( + ClickHouseBurstRateThrottle, + ClickHouseSustainedRateThrottle, +) +from posthog.utils import convert_property_value, flatten + + +class SessionViewSet( + TeamAndOrgViewSetMixin, + viewsets.ViewSet, +): + scope_object = "query" + throttle_classes = [ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle] + + @action(methods=["GET"], detail=False) + def values(self, request: request.Request, **kwargs) -> response.Response: + team = self.team + + key = request.GET.get("key") + search_term = request.GET.get("value") + + if not key: + raise ValidationError(detail=f"Key not provided") + + result = get_lazy_session_table_values(key, search_term=search_term, team=team) + + flattened = [] + for value in result: + try: + # Try loading as json for dicts or arrays + flattened.append(json.loads(value[0])) + except json.decoder.JSONDecodeError: + flattened.append(value[0]) + return response.Response([{"name": convert_property_value(value)} for value in flatten(flattened)]) + + @action(methods=["GET"], detail=False) + def property_definitions(self, request: request.Request, **kwargs) -> response.Response: + search = request.GET.get("search") + + # unlike e.g. event properties, there's a very limited number of session properties, + # so we can just return them all + results = get_lazy_session_table_properties(search) + return response.Response( + { + "count": len(results), + "results": results, + } + ) diff --git a/posthog/api/test/__snapshots__/test_api_docs.ambr b/posthog/api/test/__snapshots__/test_api_docs.ambr index b4a5bb2780673..37bb741043ef9 100644 --- a/posthog/api/test/__snapshots__/test_api_docs.ambr +++ b/posthog/api/test/__snapshots__/test_api_docs.ambr @@ -84,6 +84,8 @@ '/home/runner/work/posthog/posthog/posthog/session_recordings/session_recording_api.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer]: could not resolve field on model with path "viewed". This is likely a custom field that does some unknown magic. Maybe consider annotating the field/property? Defaulting to "string". (Exception: SessionRecording has no field named \'viewed\')', '/home/runner/work/posthog/posthog/posthog/api/person.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer > MinimalPersonSerializer]: unable to resolve type hint for function "get_distinct_ids". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/session_recordings/session_recording_api.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer]: unable to resolve type hint for function "storage". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/session.py: Error [SessionViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.', + '/home/runner/work/posthog/posthog/posthog/api/session.py: Warning [SessionViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/ee/api/subscription.py: Warning [SubscriptionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.subscription.Subscription" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/ee/api/subscription.py: Warning [SubscriptionViewSet > SubscriptionSerializer]: unable to resolve type hint for function "summary". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/survey.py: Warning [SurveyViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.feedback.survey.Survey" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', diff --git a/posthog/api/test/test_session.py b/posthog/api/test/test_session.py new file mode 100644 index 0000000000000..46fcafabd7c13 --- /dev/null +++ b/posthog/api/test/test_session.py @@ -0,0 +1,137 @@ +import uuid + +from rest_framework import status + +from posthog.models.event.util import create_event +from posthog.test.base import APIBaseTest + + +class TestSessionsAPI(APIBaseTest): + def setUp(self) -> None: + super().setUp() + + create_event( + team=self.team, + event="$pageview", + distinct_id="d1", + properties={"$session_id": "s1", "utm_source": "google"}, + event_uuid=(uuid.uuid4()), + ) + create_event( + team=self.team, + event="$pageview", + distinct_id="d1", + properties={"$session_id": "s1", "utm_source": "youtube"}, + event_uuid=(uuid.uuid4()), + ) + + def test_expected_session_properties(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/property_definitions/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_properties = {entry["name"] for entry in response.json()["results"]} + expected_properties = { + "$autocapture_count", + "$channel_type", + "$end_timestamp", + "$entry_url", + "$exit_url", + "$initial_gad_source", + "$initial_gclid", + "$initial_referring_domain", + "$initial_utm_campaign", + "$initial_utm_content", + "$initial_utm_medium", + "$initial_utm_source", + "$initial_utm_term", + "$pageview_count", + "$session_duration", + "$start_timestamp", + } + assert actual_properties == expected_properties + + def test_search_session_properties(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/property_definitions/?search=utm") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_properties = {entry["name"] for entry in response.json()["results"]} + expected_properties = { + "$initial_utm_campaign", + "$initial_utm_content", + "$initial_utm_medium", + "$initial_utm_source", + "$initial_utm_term", + } + assert actual_properties == expected_properties + + def test_empty_search_session_properties(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/property_definitions/?search=doesnotexist") + self.assertEqual(response.status_code, status.HTTP_200_OK) + assert len(response.json()["results"]) == 0 + + def test_list_channel_type_values(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/values/?key=$channel_type") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_values = {entry["name"] for entry in response.json()} + expected_values = { + "Affiliate", + "Audio", + "Cross Network", + "Direct", + "Email", + "Organic Search", + "Organic Shopping", + "Organic Video", + "Other", + "Paid Other", + "Paid Search", + "Paid Shopping", + "Paid Video", + "Push", + "Referral", + "SMS", + } + assert actual_values == expected_values + + def test_search_channel_type_values(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/values/?key=$channel_type&value=paid") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_values = {entry["name"] for entry in response.json()} + expected_values = { + "Paid Other", + "Paid Search", + "Paid Shopping", + "Paid Video", + } + assert actual_values == expected_values + + def test_list_session_property_values(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/values/?key=$initial_utm_source") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_values = {entry["name"] for entry in response.json()} + expected_values = { + "google", + "youtube", + } + assert actual_values == expected_values + + def test_search_session_property_values(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/values/?key=$initial_utm_source&value=tub") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_values = {entry["name"] for entry in response.json()} + expected_values = { + "youtube", + } + assert actual_values == expected_values + + def test_search_session_property_no_matching_values(self): + response = self.client.get( + f"/api/projects/{self.team.pk}/sessions/values/?key=$initial_utm_source&value=doesnotexist" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + assert len(response.json()) == 0 + + def test_search_missing_session_property_values(self): + response = self.client.get( + f"/api/projects/{self.team.pk}/sessions/values/?key=$initial_utm_source&value=doesnotexist" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + assert len(response.json()) == 0 diff --git a/posthog/hogql/database/schema/channel_type.py b/posthog/hogql/database/schema/channel_type.py index 39c9b31d36918..c45c71458d7d1 100644 --- a/posthog/hogql/database/schema/channel_type.py +++ b/posthog/hogql/database/schema/channel_type.py @@ -139,3 +139,23 @@ def wrap_with_null_if_empty(expr: ast.Expr) -> ast.Expr: "gad_source": wrap_with_null_if_empty(gad_source), }, ) + + +POSSIBLE_CHANNEL_TYPES = [ + "Cross Network", + "Paid Search", + "Paid Video", + "Paid Shopping", + "Paid Other", + "Direct", + "Organic Search", + "Organic Video", + "Organic Shopping", + "Push", + "SMS", + "Audio", + "Email", + "Referral", + "Affiliate", + "Other", +] diff --git a/posthog/hogql/database/schema/sessions.py b/posthog/hogql/database/schema/sessions.py index 0bd6bfef09caf..63f0e4f98e79f 100644 --- a/posthog/hogql/database/schema/sessions.py +++ b/posthog/hogql/database/schema/sessions.py @@ -1,4 +1,4 @@ -from typing import cast, Any, TYPE_CHECKING +from typing import cast, Any, Optional, TYPE_CHECKING from posthog.hogql import ast from posthog.hogql.context import HogQLContext @@ -11,13 +11,21 @@ StringArrayDatabaseField, DatabaseField, LazyTable, + FloatDatabaseField, + BooleanDatabaseField, ) -from posthog.hogql.database.schema.channel_type import create_channel_type_expr +from posthog.hogql.database.schema.channel_type import create_channel_type_expr, POSSIBLE_CHANNEL_TYPES from posthog.hogql.database.schema.util.session_where_clause_extractor import SessionMinTimestampWhereClauseExtractor from posthog.hogql.errors import ResolutionError +from posthog.models.property_definition import PropertyType +from posthog.models.sessions.sql import ( + SELECT_SESSION_PROP_STRING_VALUES_SQL_WITH_FILTER, + SELECT_SESSION_PROP_STRING_VALUES_SQL, +) +from posthog.queries.insight import insight_sync_execute if TYPE_CHECKING: - pass + from posthog.models.team import Team RAW_SESSIONS_FIELDS: dict[str, FieldOrTable] = { "id": StringDatabaseField(name="session_id"), @@ -200,6 +208,11 @@ def to_printed_clickhouse(self, context): def to_printed_hogql(self): return "sessions" + def avoid_asterisk_fields(self) -> list[str]: + return [ + "duration", # alias of $session_duration, deprecated but included for backwards compatibility + ] + def join_events_table_to_sessions_table( from_table: str, to_table: str, requested_fields: dict[str, Any], context: HogQLContext, node: ast.SelectQuery @@ -220,3 +233,97 @@ def join_events_table_to_sessions_table( ) ) return join_expr + + +def get_lazy_session_table_properties(search: Optional[str]): + # some fields shouldn't appear as properties + hidden_fields = {"team_id", "distinct_id", "session_id", "id", "$event_count_map", "$urls", "duration"} + + # some fields should have a specific property type which isn't derivable from the type of database field + property_type_overrides = { + "$session_duration": PropertyType.Duration, + } + + def get_property_type(field_name: str, field_definition: FieldOrTable): + if field_name in property_type_overrides: + return property_type_overrides[field_name] + if isinstance(field_definition, IntegerDatabaseField) or isinstance(field_definition, FloatDatabaseField): + return PropertyType.Numeric + if isinstance(field_definition, DateTimeDatabaseField): + return PropertyType.Datetime + if isinstance(field_definition, BooleanDatabaseField): + return PropertyType.Boolean + return PropertyType.String + + results = [ + { + "id": field_name, + "name": field_name, + "is_numerical": isinstance(field_definition, IntegerDatabaseField) + or isinstance(field_definition, FloatDatabaseField), + "property_type": get_property_type(field_name, field_definition), + "is_seen_on_filtered_events": None, + "tags": [], + } + for field_name, field_definition in LAZY_SESSIONS_FIELDS.items() + if (not search or search.lower() in field_name.lower()) and field_name not in hidden_fields + ] + return results + + +SESSION_PROPERTY_TO_RAW_SESSIONS_EXPR_MAP = { + "$initial_referring_domain": "finalizeAggregation(initial_referring_domain)", + "$initial_utm_source": "finalizeAggregation(initial_utm_source)", + "$initial_utm_campaign": "finalizeAggregation(initial_utm_campaign)", + "$initial_utm_medium": "finalizeAggregation(initial_utm_medium)", + "$initial_utm_term": "finalizeAggregation(initial_utm_term)", + "$initial_utm_content": "finalizeAggregation(initial_utm_content)", + "$initial_gclid": "finalizeAggregation(initial_gclid)", + "$initial_gad_source": "finalizeAggregation(initial_gad_source)", + "$initial_gclsrc": "finalizeAggregation(initial_gclsrc)", + "$initial_dclid": "finalizeAggregation(initial_dclid)", + "$initial_gbraid": "finalizeAggregation(initial_gbraid)", + "$initial_wbraid": "finalizeAggregation(initial_wbraid)", + "$initial_fbclid": "finalizeAggregation(initial_fbclid)", + "$initial_msclkid": "finalizeAggregation(initial_msclkid)", + "$initial_twclid": "finalizeAggregation(initial_twclid)", + "$initial_li_fat_id": "finalizeAggregation(initial_li_fat_id)", + "$initial_mc_cid": "finalizeAggregation(initial_mc_cid)", + "$initial_igshid": "finalizeAggregation(initial_igshid)", + "$initial_ttclid": "finalizeAggregation(initial_ttclid)", + "$entry_url": "finalizeAggregation(entry_url)", + "$exit_url": "finalizeAggregation(exit_url)", +} + + +def get_lazy_session_table_values(key: str, search_term: Optional[str], team: "Team"): + # the sessions table does not have a properties json object like the events and person tables + + if key == "$channel_type": + return [[name] for name in POSSIBLE_CHANNEL_TYPES if not search_term or search_term.lower() in name.lower()] + + expr = SESSION_PROPERTY_TO_RAW_SESSIONS_EXPR_MAP.get(key) + + if not expr: + return [] + + field_definition = LAZY_SESSIONS_FIELDS.get(key) + if not field_definition: + return [] + + if isinstance(field_definition, StringDatabaseField): + if search_term: + return insight_sync_execute( + SELECT_SESSION_PROP_STRING_VALUES_SQL_WITH_FILTER.format(property_expr=expr), + {"team_id": team.pk, "key": key, "value": "%{}%".format(search_term)}, + query_type="get_session_property_values_with_value", + team_id=team.pk, + ) + return insight_sync_execute( + SELECT_SESSION_PROP_STRING_VALUES_SQL.format(property_expr=expr), + {"team_id": team.pk, "key": key}, + query_type="get_session_property_values", + team_id=team.pk, + ) + + return [] diff --git a/posthog/hogql/database/schema/test/test_sessions.py b/posthog/hogql/database/schema/test/test_sessions.py index 2f4728fb9b558..230ffc1bc1897 100644 --- a/posthog/hogql/database/schema/test/test_sessions.py +++ b/posthog/hogql/database/schema/test/test_sessions.py @@ -5,6 +5,7 @@ APIBaseTest, ClickhouseTestMixin, _create_event, + _create_person, ) @@ -103,3 +104,35 @@ def test_events_session_dot_channel_type(self): result[0], "Paid Search", ) + + def test_persons_and_sessions_on_events(self): + p1 = _create_person(distinct_ids=["d1"], team=self.team) + p2 = _create_person(distinct_ids=["d2"], team=self.team) + + s1 = "session_test_persons_and_sessions_on_events_1" + s2 = "session_test_persons_and_sessions_on_events_2" + + _create_event( + event="$pageview", + team=self.team, + distinct_id="d1", + properties={"$session_id": s1, "utm_source": "source1"}, + ) + _create_event( + event="$pageview", + team=self.team, + distinct_id="d2", + properties={"$session_id": s2, "utm_source": "source2"}, + ) + + response = execute_hogql_query( + parse_select( + "select events.person_id, session.$initial_utm_source from events where $session_id = {session_id} or $session_id = {session_id2} order by 2 asc", + placeholders={"session_id": ast.Constant(value=s1), "session_id2": ast.Constant(value=s2)}, + ), + self.team, + ) + + [row1, row2] = response.results or [] + self.assertEqual(row1, (p1.uuid, "source1")) + self.assertEqual(row2, (p2.uuid, "source2")) diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py index fb1288ac1bd26..cd6a218e7ea0f 100644 --- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py +++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py @@ -24,6 +24,7 @@ WebStatsTableQuery, PersonPropertyFilter, SamplingRate, + SessionPropertyFilter, ) from posthog.utils import generate_cache_key, get_safe_cache @@ -51,7 +52,9 @@ def pathname_property_filter(self) -> Optional[EventPropertyFilter]: return None @cached_property - def property_filters_without_pathname(self) -> list[Union[EventPropertyFilter, PersonPropertyFilter]]: + def property_filters_without_pathname( + self, + ) -> list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]]: return [p for p in self.query.properties if p.key != "$pathname"] def session_where(self, include_previous_period: Optional[bool] = None): diff --git a/posthog/models/property/util.py b/posthog/models/property/util.py index b1ce9b6087aaa..de2602539e6ec 100644 --- a/posthog/models/property/util.py +++ b/posthog/models/property/util.py @@ -930,7 +930,7 @@ def get_session_property_filter_statement(prop: Property, idx: int, prepend: str ) else: - raise exceptions.ValidationError(f"Property '{prop.key}' is not allowed in session property filters.") + raise exceptions.ValidationError(f"Session property '{prop.key}' is only valid in HogQL queries.") def clear_excess_levels(prop: Union["PropertyGroup", "Property"], skip=False): diff --git a/posthog/models/sessions/sql.py b/posthog/models/sessions/sql.py index 6bebc73e023f4..22d3431099f94 100644 --- a/posthog/models/sessions/sql.py +++ b/posthog/models/sessions/sql.py @@ -260,3 +260,44 @@ def source_column(column_name: str) -> str: GROUP BY session_id, team_id """ ) + +SELECT_SESSION_PROP_STRING_VALUES_SQL = """ +SELECT + value, + count(value) +FROM ( + SELECT + {property_expr} as value + FROM + sessions + WHERE + team_id = %(team_id)s AND + {property_expr} IS NOT NULL AND + {property_expr} != '' + ORDER BY session_id DESC + LIMIT 100000 +) +GROUP BY value +ORDER BY count(value) DESC +LIMIT 20 +""" + +SELECT_SESSION_PROP_STRING_VALUES_SQL_WITH_FILTER = """ +SELECT + value, + count(value) +FROM ( + SELECT + {property_expr} as value + FROM + sessions + WHERE + team_id = %(team_id)s AND + {property_expr} ILIKE %(value)s + ORDER BY session_id DESC + LIMIT 100000 +) +GROUP BY value +ORDER BY count(value) DESC +LIMIT 20 +""" diff --git a/posthog/schema.py b/posthog/schema.py index 46ad0beb9a11a..281cd8ef3a039 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -1699,7 +1699,7 @@ class WebAnalyticsQueryBase(BaseModel): ) dateRange: Optional[DateRange] = None modifiers: Optional[HogQLQueryModifiers] = None - properties: list[Union[EventPropertyFilter, PersonPropertyFilter]] + properties: list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]] sampling: Optional[Sampling] = None useSessionsTable: Optional[bool] = None @@ -1712,7 +1712,7 @@ class WebOverviewQuery(BaseModel): dateRange: Optional[DateRange] = None kind: Literal["WebOverviewQuery"] = "WebOverviewQuery" modifiers: Optional[HogQLQueryModifiers] = None - properties: list[Union[EventPropertyFilter, PersonPropertyFilter]] + properties: list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]] response: Optional[WebOverviewQueryResponse] = None sampling: Optional[Sampling] = None useSessionsTable: Optional[bool] = None @@ -1730,7 +1730,7 @@ class WebStatsTableQuery(BaseModel): kind: Literal["WebStatsTableQuery"] = "WebStatsTableQuery" limit: Optional[int] = None modifiers: Optional[HogQLQueryModifiers] = None - properties: list[Union[EventPropertyFilter, PersonPropertyFilter]] + properties: list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]] response: Optional[WebStatsTableQueryResponse] = None sampling: Optional[Sampling] = None useSessionsTable: Optional[bool] = None @@ -1743,7 +1743,7 @@ class WebTopClicksQuery(BaseModel): dateRange: Optional[DateRange] = None kind: Literal["WebTopClicksQuery"] = "WebTopClicksQuery" modifiers: Optional[HogQLQueryModifiers] = None - properties: list[Union[EventPropertyFilter, PersonPropertyFilter]] + properties: list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]] response: Optional[WebTopClicksQueryResponse] = None sampling: Optional[Sampling] = None useSessionsTable: Optional[bool] = None From bf23ec6320d792e3ad76279fa27346dc816e8e6a Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 25 Apr 2024 17:21:26 +0200 Subject: [PATCH 7/8] Revert "fix: Wrap asgi application to ignore lifetime requests" (#21854) Revert "fix: Wrap asgi application to ignore lifetime requests (#21802)" This reverts commit d2ed2ea9dfa359856356a2e38cc961b9c10d60d2. --- posthog/asgi.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/posthog/asgi.py b/posthog/asgi.py index 678645bb01d3b..22912a0c7b76e 100644 --- a/posthog/asgi.py +++ b/posthog/asgi.py @@ -1,17 +1,8 @@ import os from django.core.asgi import get_asgi_application -from django.http.response import HttpResponse os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings") os.environ.setdefault("SERVER_GATEWAY_INTERFACE", "ASGI") - -def lifetime_wrapper(func): - async def inner(scope, receive, send): - if scope["type"] != "http": - return HttpResponse(status=501) - return func(scope, receive, send) - - -application = lifetime_wrapper(get_asgi_application()) +application = get_asgi_application() From 06fe778be179c1891e6922f47f26b680e779da03 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 25 Apr 2024 09:29:21 -0600 Subject: [PATCH 8/8] chore: add CH migration to add new person_mode option (#21831) --- .../0060_person_mode_force_upgrade.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 posthog/clickhouse/migrations/0060_person_mode_force_upgrade.py diff --git a/posthog/clickhouse/migrations/0060_person_mode_force_upgrade.py b/posthog/clickhouse/migrations/0060_person_mode_force_upgrade.py new file mode 100644 index 0000000000000..7b3fa1b698740 --- /dev/null +++ b/posthog/clickhouse/migrations/0060_person_mode_force_upgrade.py @@ -0,0 +1,32 @@ +from infi.clickhouse_orm import migrations + +from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions +from posthog.client import sync_execute +from posthog.models.event.sql import ( + EVENTS_TABLE_JSON_MV_SQL, + KAFKA_EVENTS_TABLE_JSON_SQL, +) +from posthog.settings import CLICKHOUSE_CLUSTER + + +# Column was added in 0057_events_person_mode +ALTER_COLUMNS_BASE_SQL = """ +ALTER TABLE {table} +ON CLUSTER {cluster} +MODIFY COLUMN person_mode Enum8('full' = 0, 'propertyless' = 1, 'force_upgrade' = 2) +""" + + +def alter_columns_in_required_tables(_): + sync_execute(ALTER_COLUMNS_BASE_SQL.format(table="events", cluster=CLICKHOUSE_CLUSTER)) + sync_execute(ALTER_COLUMNS_BASE_SQL.format(table="writable_events", cluster=CLICKHOUSE_CLUSTER)) + sync_execute(ALTER_COLUMNS_BASE_SQL.format(table="sharded_events", cluster=CLICKHOUSE_CLUSTER)) + + +operations = [ + run_sql_with_exceptions(f"DROP TABLE IF EXISTS events_json_mv ON CLUSTER '{CLICKHOUSE_CLUSTER}'"), + run_sql_with_exceptions(f"DROP TABLE IF EXISTS kafka_events_json ON CLUSTER '{CLICKHOUSE_CLUSTER}'"), + migrations.RunPython(alter_columns_in_required_tables), + run_sql_with_exceptions(KAFKA_EVENTS_TABLE_JSON_SQL()), + run_sql_with_exceptions(EVENTS_TABLE_JSON_MV_SQL()), +]