From 4f2ec9d392c42290f3e414f09ff4b56bdc92f54e Mon Sep 17 00:00:00 2001 From: Ted Kaemming <65315+tkaemming@users.noreply.github.com> Date: Tue, 26 Mar 2024 10:31:22 -0700 Subject: [PATCH 1/7] Add setting to be able to disable capture overflow entirely. --- posthog/api/capture.py | 17 ++++++++--------- posthog/settings/ingestion.py | 4 ++++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/posthog/api/capture.py b/posthog/api/capture.py index 73998505bb822..3d67911f4a8b4 100644 --- a/posthog/api/capture.py +++ b/posthog/api/capture.py @@ -602,11 +602,6 @@ def capture_internal( token=token, ) - # We aim to always partition by {team_id}:{distinct_id} but allow - # overriding this to deal with hot partitions in specific cases. - # Setting the partition key to None means using random partitioning. - kafka_partition_key = None - if event["event"] in SESSION_RECORDING_EVENT_NAMES: session_id = event["properties"]["$session_id"] headers = [ @@ -623,13 +618,17 @@ def capture_internal( parsed_event, event["event"], partition_key=session_id, headers=headers, overflowing=overflowing ) + # We aim to always partition by {team_id}:{distinct_id} but allow + # overriding this to deal with hot partitions in specific cases. + # Setting the partition key to None means using random partitioning. candidate_partition_key = f"{token}:{distinct_id}" - if ( - distinct_id.lower() not in LIKELY_ANONYMOUS_IDS - and not is_randomly_partitioned(candidate_partition_key) - or historical + not historical + and settings.CAPTURE_OVERFLOW_ENABLED + and (distinct_id.lower() in LIKELY_ANONYMOUS_IDS or is_randomly_partitioned(candidate_partition_key)) ): + kafka_partition_key = None + else: kafka_partition_key = hashlib.sha256(candidate_partition_key.encode()).hexdigest() return log_event(parsed_event, event["event"], partition_key=kafka_partition_key, historical=historical) diff --git a/posthog/settings/ingestion.py b/posthog/settings/ingestion.py index 2bc532ac9cd92..3b4d039a25b20 100644 --- a/posthog/settings/ingestion.py +++ b/posthog/settings/ingestion.py @@ -11,6 +11,10 @@ # KEEP IN SYNC WITH plugin-server/src/config/config.ts BUFFER_CONVERSION_SECONDS = get_from_env("BUFFER_CONVERSION_SECONDS", default=60, type_cast=int) +# Whether or not overflow (random partitioning) should be enabled *at all*. +# Note that this setting takes precedence over other overflow-related settings +# below, if disabled. +CAPTURE_OVERFLOW_ENABLED = get_from_env("CAPTURE_OVERFLOW_ENABLED", True, type_cast=str_to_bool) # A list of pairs (in the format 2:myLovelyId) that we should use # random partitioning for when producing events to the Kafka topic consumed by the plugin server. From d0c6aefe8794fc479ad1c65236f501b4ff793750 Mon Sep 17 00:00:00 2001 From: Ted Kaemming <65315+tkaemming@users.noreply.github.com> Date: Tue, 26 Mar 2024 10:42:41 -0700 Subject: [PATCH 2/7] Better name for the setting as this is sort of decoupled from overflow (but also not really) --- posthog/api/capture.py | 2 +- posthog/settings/ingestion.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/posthog/api/capture.py b/posthog/api/capture.py index 3d67911f4a8b4..6b921dd27ea48 100644 --- a/posthog/api/capture.py +++ b/posthog/api/capture.py @@ -624,7 +624,7 @@ def capture_internal( candidate_partition_key = f"{token}:{distinct_id}" if ( not historical - and settings.CAPTURE_OVERFLOW_ENABLED + and settings.CAPTURE_ALLOW_RANDOM_PARTITIONING and (distinct_id.lower() in LIKELY_ANONYMOUS_IDS or is_randomly_partitioned(candidate_partition_key)) ): kafka_partition_key = None diff --git a/posthog/settings/ingestion.py b/posthog/settings/ingestion.py index 3b4d039a25b20..efbe05371d5e3 100644 --- a/posthog/settings/ingestion.py +++ b/posthog/settings/ingestion.py @@ -11,10 +11,12 @@ # KEEP IN SYNC WITH plugin-server/src/config/config.ts BUFFER_CONVERSION_SECONDS = get_from_env("BUFFER_CONVERSION_SECONDS", default=60, type_cast=int) -# Whether or not overflow (random partitioning) should be enabled *at all*. -# Note that this setting takes precedence over other overflow-related settings -# below, if disabled. -CAPTURE_OVERFLOW_ENABLED = get_from_env("CAPTURE_OVERFLOW_ENABLED", True, type_cast=str_to_bool) +# Whether or not random partitioning (i.e. overflow routing) should be enabled +# at all. Enabling this setting this does not cause messages to be randomly +# partitioned, but only allows them to be be randomly partitioned if other +# criteria are met. Note that this setting takes precedence over other +# partition-related settings below, if disabled. +CAPTURE_ALLOW_RANDOM_PARTITIONING = get_from_env("CAPTURE_ALLOW_RANDOM_PARTITIONING", True, type_cast=str_to_bool) # A list of pairs (in the format 2:myLovelyId) that we should use # random partitioning for when producing events to the Kafka topic consumed by the plugin server. From db47485b6dd3cff20b9ca5453957416f3680c5a3 Mon Sep 17 00:00:00 2001 From: Ted Kaemming <65315+tkaemming@users.noreply.github.com> Date: Tue, 26 Mar 2024 10:51:21 -0700 Subject: [PATCH 3/7] Reword comment so that it's a bit clearer --- posthog/settings/ingestion.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/posthog/settings/ingestion.py b/posthog/settings/ingestion.py index efbe05371d5e3..463db4121c440 100644 --- a/posthog/settings/ingestion.py +++ b/posthog/settings/ingestion.py @@ -11,11 +11,10 @@ # KEEP IN SYNC WITH plugin-server/src/config/config.ts BUFFER_CONVERSION_SECONDS = get_from_env("BUFFER_CONVERSION_SECONDS", default=60, type_cast=int) -# Whether or not random partitioning (i.e. overflow routing) should be enabled -# at all. Enabling this setting this does not cause messages to be randomly -# partitioned, but only allows them to be be randomly partitioned if other -# criteria are met. Note that this setting takes precedence over other -# partition-related settings below, if disabled. +# Whether or not random partitioning (i.e. overflow routing) should be allowed. +# (Enabling this setting this does not cause messages to be randomly +# partitioned.) Note that this setting, if disabled, takes precedence over other +# partitioning-related settings below. CAPTURE_ALLOW_RANDOM_PARTITIONING = get_from_env("CAPTURE_ALLOW_RANDOM_PARTITIONING", True, type_cast=str_to_bool) # A list of pairs (in the format 2:myLovelyId) that we should use From d28b5beb200353f0ca6ef561513091307b75b9fc Mon Sep 17 00:00:00 2001 From: Ted Kaemming <65315+tkaemming@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:24:06 -0700 Subject: [PATCH 4/7] Extend capture test to cover setting enabled and disabled --- posthog/api/test/test_capture.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/posthog/api/test/test_capture.py b/posthog/api/test/test_capture.py index 2a80186082dea..a0fc8826c95c6 100644 --- a/posthog/api/test/test_capture.py +++ b/posthog/api/test/test_capture.py @@ -4,6 +4,7 @@ import base64 import gzip import json +from django.test import override_settings import lzstring import pathlib import pytest @@ -281,8 +282,7 @@ def test_is_randomly_partitioned(self): assert is_randomly_partitioned(override_key) is True @patch("posthog.kafka_client.client._KafkaProducer.produce") - def test_capture_randomly_partitions_with_likely_anonymous_ids(self, kafka_produce): - """Test is_randomly_partitioned in the prescence of likely anonymous ids.""" + def _do_test_capture_with_likely_anonymous_ids(self, kafka_produce, expect_random_partitioning: bool): for distinct_id in LIKELY_ANONYMOUS_IDS: data = { "event": "$autocapture", @@ -298,9 +298,23 @@ def test_capture_randomly_partitions_with_likely_anonymous_ids(self, kafka_produ ) kafka_produce.assert_called_with( - topic=KAFKA_EVENTS_PLUGIN_INGESTION_TOPIC, data=ANY, key=None, headers=None + topic=KAFKA_EVENTS_PLUGIN_INGESTION_TOPIC, + data=ANY, + key=None if expect_random_partitioning else ANY, + headers=None, ) + if not expect_random_partitioning: + assert kafka_produce.mock_calls[0].kwargs["key"] is not None + + def test_capture_randomly_partitions_with_likely_anonymous_ids(self): + """Test is_randomly_partitioned in the prescence of likely anonymous ids, if enabled.""" + with override_settings(CAPTURE_ALLOW_RANDOM_PARTITIONING=True): + self._do_test_capture_with_likely_anonymous_ids(expect_random_partitioning=True) + + with override_settings(CAPTURE_ALLOW_RANDOM_PARTITIONING=False): + self._do_test_capture_with_likely_anonymous_ids(expect_random_partitioning=False) + def test_cached_is_randomly_partitioned(self): """Assert the behavior of is_randomly_partitioned under certain cache settings. From 068991f5e626b63805b4a97bae16014bb8b86bf0 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 23:34:15 +0000 Subject: [PATCH 5/7] Update query snapshots --- .../test/__snapshots__/test_annotation.ambr | 250 +++++++----------- 1 file changed, 91 insertions(+), 159 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_annotation.ambr b/posthog/api/test/__snapshots__/test_annotation.ambr index b75f8a9661f7e..3bac87bc32080 100644 --- a/posthog/api/test/__snapshots__/test_annotation.ambr +++ b/posthog/api/test/__snapshots__/test_annotation.ambr @@ -84,6 +84,85 @@ ''' # --- # name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.10 + ''' + SELECT "posthog_annotation"."id", + "posthog_annotation"."content", + "posthog_annotation"."created_at", + "posthog_annotation"."updated_at", + "posthog_annotation"."dashboard_item_id", + "posthog_annotation"."team_id", + "posthog_annotation"."organization_id", + "posthog_annotation"."created_by_id", + "posthog_annotation"."scope", + "posthog_annotation"."creation_type", + "posthog_annotation"."date_marker", + "posthog_annotation"."deleted", + "posthog_annotation"."apply_all", + "posthog_dashboarditem"."id", + "posthog_dashboarditem"."name", + "posthog_dashboarditem"."derived_name", + "posthog_dashboarditem"."description", + "posthog_dashboarditem"."team_id", + "posthog_dashboarditem"."filters", + "posthog_dashboarditem"."filters_hash", + "posthog_dashboarditem"."query", + "posthog_dashboarditem"."order", + "posthog_dashboarditem"."deleted", + "posthog_dashboarditem"."saved", + "posthog_dashboarditem"."created_at", + "posthog_dashboarditem"."last_refresh", + "posthog_dashboarditem"."refreshing", + "posthog_dashboarditem"."created_by_id", + "posthog_dashboarditem"."is_sample", + "posthog_dashboarditem"."short_id", + "posthog_dashboarditem"."favorited", + "posthog_dashboarditem"."refresh_attempt", + "posthog_dashboarditem"."last_modified_at", + "posthog_dashboarditem"."last_modified_by_id", + "posthog_dashboarditem"."dashboard_id", + "posthog_dashboarditem"."layouts", + "posthog_dashboarditem"."color", + "posthog_dashboarditem"."dive_dashboard_id", + "posthog_dashboarditem"."updated_at", + "posthog_dashboarditem"."deprecated_tags", + "posthog_dashboarditem"."tags", + "posthog_user"."id", + "posthog_user"."password", + "posthog_user"."last_login", + "posthog_user"."first_name", + "posthog_user"."last_name", + "posthog_user"."is_staff", + "posthog_user"."is_active", + "posthog_user"."date_joined", + "posthog_user"."uuid", + "posthog_user"."current_organization_id", + "posthog_user"."current_team_id", + "posthog_user"."email", + "posthog_user"."pending_email", + "posthog_user"."temporary_token", + "posthog_user"."distinct_id", + "posthog_user"."is_email_verified", + "posthog_user"."requested_password_reset_at", + "posthog_user"."has_seen_product_intro_for", + "posthog_user"."strapi_id", + "posthog_user"."email_opt_in", + "posthog_user"."theme_mode", + "posthog_user"."partial_notification_settings", + "posthog_user"."anonymize_data", + "posthog_user"."toolbar_mode", + "posthog_user"."events_column_config" + FROM "posthog_annotation" + LEFT OUTER JOIN "posthog_dashboarditem" ON ("posthog_annotation"."dashboard_item_id" = "posthog_dashboarditem"."id") + LEFT OUTER JOIN "posthog_user" ON ("posthog_annotation"."created_by_id" = "posthog_user"."id") + WHERE ((("posthog_annotation"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid + AND "posthog_annotation"."scope" = 'organization') + OR "posthog_annotation"."team_id" = 2) + AND NOT "posthog_annotation"."deleted") + ORDER BY "posthog_annotation"."date_marker" DESC + LIMIT 1000 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ + ''' +# --- +# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.11 ''' SELECT "posthog_user"."id", "posthog_user"."password", @@ -114,7 +193,7 @@ LIMIT 21 /**/ ''' # --- -# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.11 +# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.12 ''' SELECT "posthog_team"."id", "posthog_team"."uuid", @@ -167,7 +246,7 @@ LIMIT 21 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- -# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.12 +# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.13 ''' SELECT "posthog_organizationmembership"."id", "posthog_organizationmembership"."organization_id", @@ -199,7 +278,7 @@ WHERE "posthog_organizationmembership"."user_id" = 2 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- -# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.13 +# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.14 ''' SELECT COUNT(*) AS "__count" FROM "posthog_annotation" @@ -209,85 +288,6 @@ AND NOT "posthog_annotation"."deleted") /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- -# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.14 - ''' - SELECT "posthog_annotation"."id", - "posthog_annotation"."content", - "posthog_annotation"."created_at", - "posthog_annotation"."updated_at", - "posthog_annotation"."dashboard_item_id", - "posthog_annotation"."team_id", - "posthog_annotation"."organization_id", - "posthog_annotation"."created_by_id", - "posthog_annotation"."scope", - "posthog_annotation"."creation_type", - "posthog_annotation"."date_marker", - "posthog_annotation"."deleted", - "posthog_annotation"."apply_all", - "posthog_dashboarditem"."id", - "posthog_dashboarditem"."name", - "posthog_dashboarditem"."derived_name", - "posthog_dashboarditem"."description", - "posthog_dashboarditem"."team_id", - "posthog_dashboarditem"."filters", - "posthog_dashboarditem"."filters_hash", - "posthog_dashboarditem"."query", - "posthog_dashboarditem"."order", - "posthog_dashboarditem"."deleted", - "posthog_dashboarditem"."saved", - "posthog_dashboarditem"."created_at", - "posthog_dashboarditem"."last_refresh", - "posthog_dashboarditem"."refreshing", - "posthog_dashboarditem"."created_by_id", - "posthog_dashboarditem"."is_sample", - "posthog_dashboarditem"."short_id", - "posthog_dashboarditem"."favorited", - "posthog_dashboarditem"."refresh_attempt", - "posthog_dashboarditem"."last_modified_at", - "posthog_dashboarditem"."last_modified_by_id", - "posthog_dashboarditem"."dashboard_id", - "posthog_dashboarditem"."layouts", - "posthog_dashboarditem"."color", - "posthog_dashboarditem"."dive_dashboard_id", - "posthog_dashboarditem"."updated_at", - "posthog_dashboarditem"."deprecated_tags", - "posthog_dashboarditem"."tags", - "posthog_user"."id", - "posthog_user"."password", - "posthog_user"."last_login", - "posthog_user"."first_name", - "posthog_user"."last_name", - "posthog_user"."is_staff", - "posthog_user"."is_active", - "posthog_user"."date_joined", - "posthog_user"."uuid", - "posthog_user"."current_organization_id", - "posthog_user"."current_team_id", - "posthog_user"."email", - "posthog_user"."pending_email", - "posthog_user"."temporary_token", - "posthog_user"."distinct_id", - "posthog_user"."is_email_verified", - "posthog_user"."requested_password_reset_at", - "posthog_user"."has_seen_product_intro_for", - "posthog_user"."strapi_id", - "posthog_user"."email_opt_in", - "posthog_user"."theme_mode", - "posthog_user"."partial_notification_settings", - "posthog_user"."anonymize_data", - "posthog_user"."toolbar_mode", - "posthog_user"."events_column_config" - FROM "posthog_annotation" - LEFT OUTER JOIN "posthog_dashboarditem" ON ("posthog_annotation"."dashboard_item_id" = "posthog_dashboarditem"."id") - LEFT OUTER JOIN "posthog_user" ON ("posthog_annotation"."created_by_id" = "posthog_user"."id") - WHERE ((("posthog_annotation"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid - AND "posthog_annotation"."scope" = 'organization') - OR "posthog_annotation"."team_id" = 2) - AND NOT "posthog_annotation"."deleted") - ORDER BY "posthog_annotation"."date_marker" DESC - LIMIT 1000 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ - ''' -# --- # name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.15 ''' SELECT "posthog_annotation"."id", @@ -538,90 +538,22 @@ # --- # name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.8 ''' - SELECT COUNT(*) AS "__count" - FROM "posthog_annotation" - WHERE ((("posthog_annotation"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid - AND "posthog_annotation"."scope" = 'organization') - OR "posthog_annotation"."team_id" = 2) - AND NOT "posthog_annotation"."deleted") /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ + SELECT "posthog_instancesetting"."id", + "posthog_instancesetting"."key", + "posthog_instancesetting"."raw_value" + FROM "posthog_instancesetting" + WHERE "posthog_instancesetting"."key" = 'constance:posthog:RATE_LIMIT_ENABLED' + ORDER BY "posthog_instancesetting"."id" ASC + LIMIT 1 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- # name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.9 ''' - SELECT "posthog_annotation"."id", - "posthog_annotation"."content", - "posthog_annotation"."created_at", - "posthog_annotation"."updated_at", - "posthog_annotation"."dashboard_item_id", - "posthog_annotation"."team_id", - "posthog_annotation"."organization_id", - "posthog_annotation"."created_by_id", - "posthog_annotation"."scope", - "posthog_annotation"."creation_type", - "posthog_annotation"."date_marker", - "posthog_annotation"."deleted", - "posthog_annotation"."apply_all", - "posthog_dashboarditem"."id", - "posthog_dashboarditem"."name", - "posthog_dashboarditem"."derived_name", - "posthog_dashboarditem"."description", - "posthog_dashboarditem"."team_id", - "posthog_dashboarditem"."filters", - "posthog_dashboarditem"."filters_hash", - "posthog_dashboarditem"."query", - "posthog_dashboarditem"."order", - "posthog_dashboarditem"."deleted", - "posthog_dashboarditem"."saved", - "posthog_dashboarditem"."created_at", - "posthog_dashboarditem"."last_refresh", - "posthog_dashboarditem"."refreshing", - "posthog_dashboarditem"."created_by_id", - "posthog_dashboarditem"."is_sample", - "posthog_dashboarditem"."short_id", - "posthog_dashboarditem"."favorited", - "posthog_dashboarditem"."refresh_attempt", - "posthog_dashboarditem"."last_modified_at", - "posthog_dashboarditem"."last_modified_by_id", - "posthog_dashboarditem"."dashboard_id", - "posthog_dashboarditem"."layouts", - "posthog_dashboarditem"."color", - "posthog_dashboarditem"."dive_dashboard_id", - "posthog_dashboarditem"."updated_at", - "posthog_dashboarditem"."deprecated_tags", - "posthog_dashboarditem"."tags", - "posthog_user"."id", - "posthog_user"."password", - "posthog_user"."last_login", - "posthog_user"."first_name", - "posthog_user"."last_name", - "posthog_user"."is_staff", - "posthog_user"."is_active", - "posthog_user"."date_joined", - "posthog_user"."uuid", - "posthog_user"."current_organization_id", - "posthog_user"."current_team_id", - "posthog_user"."email", - "posthog_user"."pending_email", - "posthog_user"."temporary_token", - "posthog_user"."distinct_id", - "posthog_user"."is_email_verified", - "posthog_user"."requested_password_reset_at", - "posthog_user"."has_seen_product_intro_for", - "posthog_user"."strapi_id", - "posthog_user"."email_opt_in", - "posthog_user"."theme_mode", - "posthog_user"."partial_notification_settings", - "posthog_user"."anonymize_data", - "posthog_user"."toolbar_mode", - "posthog_user"."events_column_config" + SELECT COUNT(*) AS "__count" FROM "posthog_annotation" - LEFT OUTER JOIN "posthog_dashboarditem" ON ("posthog_annotation"."dashboard_item_id" = "posthog_dashboarditem"."id") - LEFT OUTER JOIN "posthog_user" ON ("posthog_annotation"."created_by_id" = "posthog_user"."id") WHERE ((("posthog_annotation"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid AND "posthog_annotation"."scope" = 'organization') OR "posthog_annotation"."team_id" = 2) - AND NOT "posthog_annotation"."deleted") - ORDER BY "posthog_annotation"."date_marker" DESC - LIMIT 1000 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ + AND NOT "posthog_annotation"."deleted") /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- From 74d194bb5f2f7adf56b5e50ee378a4d4c7bbef13 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 23:51:09 +0000 Subject: [PATCH 6/7] Update query snapshots --- .../test/__snapshots__/test_annotation.ambr | 250 +++++++++++------- 1 file changed, 159 insertions(+), 91 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_annotation.ambr b/posthog/api/test/__snapshots__/test_annotation.ambr index 3bac87bc32080..b75f8a9661f7e 100644 --- a/posthog/api/test/__snapshots__/test_annotation.ambr +++ b/posthog/api/test/__snapshots__/test_annotation.ambr @@ -84,85 +84,6 @@ ''' # --- # name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.10 - ''' - SELECT "posthog_annotation"."id", - "posthog_annotation"."content", - "posthog_annotation"."created_at", - "posthog_annotation"."updated_at", - "posthog_annotation"."dashboard_item_id", - "posthog_annotation"."team_id", - "posthog_annotation"."organization_id", - "posthog_annotation"."created_by_id", - "posthog_annotation"."scope", - "posthog_annotation"."creation_type", - "posthog_annotation"."date_marker", - "posthog_annotation"."deleted", - "posthog_annotation"."apply_all", - "posthog_dashboarditem"."id", - "posthog_dashboarditem"."name", - "posthog_dashboarditem"."derived_name", - "posthog_dashboarditem"."description", - "posthog_dashboarditem"."team_id", - "posthog_dashboarditem"."filters", - "posthog_dashboarditem"."filters_hash", - "posthog_dashboarditem"."query", - "posthog_dashboarditem"."order", - "posthog_dashboarditem"."deleted", - "posthog_dashboarditem"."saved", - "posthog_dashboarditem"."created_at", - "posthog_dashboarditem"."last_refresh", - "posthog_dashboarditem"."refreshing", - "posthog_dashboarditem"."created_by_id", - "posthog_dashboarditem"."is_sample", - "posthog_dashboarditem"."short_id", - "posthog_dashboarditem"."favorited", - "posthog_dashboarditem"."refresh_attempt", - "posthog_dashboarditem"."last_modified_at", - "posthog_dashboarditem"."last_modified_by_id", - "posthog_dashboarditem"."dashboard_id", - "posthog_dashboarditem"."layouts", - "posthog_dashboarditem"."color", - "posthog_dashboarditem"."dive_dashboard_id", - "posthog_dashboarditem"."updated_at", - "posthog_dashboarditem"."deprecated_tags", - "posthog_dashboarditem"."tags", - "posthog_user"."id", - "posthog_user"."password", - "posthog_user"."last_login", - "posthog_user"."first_name", - "posthog_user"."last_name", - "posthog_user"."is_staff", - "posthog_user"."is_active", - "posthog_user"."date_joined", - "posthog_user"."uuid", - "posthog_user"."current_organization_id", - "posthog_user"."current_team_id", - "posthog_user"."email", - "posthog_user"."pending_email", - "posthog_user"."temporary_token", - "posthog_user"."distinct_id", - "posthog_user"."is_email_verified", - "posthog_user"."requested_password_reset_at", - "posthog_user"."has_seen_product_intro_for", - "posthog_user"."strapi_id", - "posthog_user"."email_opt_in", - "posthog_user"."theme_mode", - "posthog_user"."partial_notification_settings", - "posthog_user"."anonymize_data", - "posthog_user"."toolbar_mode", - "posthog_user"."events_column_config" - FROM "posthog_annotation" - LEFT OUTER JOIN "posthog_dashboarditem" ON ("posthog_annotation"."dashboard_item_id" = "posthog_dashboarditem"."id") - LEFT OUTER JOIN "posthog_user" ON ("posthog_annotation"."created_by_id" = "posthog_user"."id") - WHERE ((("posthog_annotation"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid - AND "posthog_annotation"."scope" = 'organization') - OR "posthog_annotation"."team_id" = 2) - AND NOT "posthog_annotation"."deleted") - ORDER BY "posthog_annotation"."date_marker" DESC - LIMIT 1000 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ - ''' -# --- -# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.11 ''' SELECT "posthog_user"."id", "posthog_user"."password", @@ -193,7 +114,7 @@ LIMIT 21 /**/ ''' # --- -# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.12 +# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.11 ''' SELECT "posthog_team"."id", "posthog_team"."uuid", @@ -246,7 +167,7 @@ LIMIT 21 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- -# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.13 +# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.12 ''' SELECT "posthog_organizationmembership"."id", "posthog_organizationmembership"."organization_id", @@ -278,7 +199,7 @@ WHERE "posthog_organizationmembership"."user_id" = 2 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- -# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.14 +# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.13 ''' SELECT COUNT(*) AS "__count" FROM "posthog_annotation" @@ -288,6 +209,85 @@ AND NOT "posthog_annotation"."deleted") /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- +# name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.14 + ''' + SELECT "posthog_annotation"."id", + "posthog_annotation"."content", + "posthog_annotation"."created_at", + "posthog_annotation"."updated_at", + "posthog_annotation"."dashboard_item_id", + "posthog_annotation"."team_id", + "posthog_annotation"."organization_id", + "posthog_annotation"."created_by_id", + "posthog_annotation"."scope", + "posthog_annotation"."creation_type", + "posthog_annotation"."date_marker", + "posthog_annotation"."deleted", + "posthog_annotation"."apply_all", + "posthog_dashboarditem"."id", + "posthog_dashboarditem"."name", + "posthog_dashboarditem"."derived_name", + "posthog_dashboarditem"."description", + "posthog_dashboarditem"."team_id", + "posthog_dashboarditem"."filters", + "posthog_dashboarditem"."filters_hash", + "posthog_dashboarditem"."query", + "posthog_dashboarditem"."order", + "posthog_dashboarditem"."deleted", + "posthog_dashboarditem"."saved", + "posthog_dashboarditem"."created_at", + "posthog_dashboarditem"."last_refresh", + "posthog_dashboarditem"."refreshing", + "posthog_dashboarditem"."created_by_id", + "posthog_dashboarditem"."is_sample", + "posthog_dashboarditem"."short_id", + "posthog_dashboarditem"."favorited", + "posthog_dashboarditem"."refresh_attempt", + "posthog_dashboarditem"."last_modified_at", + "posthog_dashboarditem"."last_modified_by_id", + "posthog_dashboarditem"."dashboard_id", + "posthog_dashboarditem"."layouts", + "posthog_dashboarditem"."color", + "posthog_dashboarditem"."dive_dashboard_id", + "posthog_dashboarditem"."updated_at", + "posthog_dashboarditem"."deprecated_tags", + "posthog_dashboarditem"."tags", + "posthog_user"."id", + "posthog_user"."password", + "posthog_user"."last_login", + "posthog_user"."first_name", + "posthog_user"."last_name", + "posthog_user"."is_staff", + "posthog_user"."is_active", + "posthog_user"."date_joined", + "posthog_user"."uuid", + "posthog_user"."current_organization_id", + "posthog_user"."current_team_id", + "posthog_user"."email", + "posthog_user"."pending_email", + "posthog_user"."temporary_token", + "posthog_user"."distinct_id", + "posthog_user"."is_email_verified", + "posthog_user"."requested_password_reset_at", + "posthog_user"."has_seen_product_intro_for", + "posthog_user"."strapi_id", + "posthog_user"."email_opt_in", + "posthog_user"."theme_mode", + "posthog_user"."partial_notification_settings", + "posthog_user"."anonymize_data", + "posthog_user"."toolbar_mode", + "posthog_user"."events_column_config" + FROM "posthog_annotation" + LEFT OUTER JOIN "posthog_dashboarditem" ON ("posthog_annotation"."dashboard_item_id" = "posthog_dashboarditem"."id") + LEFT OUTER JOIN "posthog_user" ON ("posthog_annotation"."created_by_id" = "posthog_user"."id") + WHERE ((("posthog_annotation"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid + AND "posthog_annotation"."scope" = 'organization') + OR "posthog_annotation"."team_id" = 2) + AND NOT "posthog_annotation"."deleted") + ORDER BY "posthog_annotation"."date_marker" DESC + LIMIT 1000 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ + ''' +# --- # name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.15 ''' SELECT "posthog_annotation"."id", @@ -538,22 +538,90 @@ # --- # name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.8 ''' - SELECT "posthog_instancesetting"."id", - "posthog_instancesetting"."key", - "posthog_instancesetting"."raw_value" - FROM "posthog_instancesetting" - WHERE "posthog_instancesetting"."key" = 'constance:posthog:RATE_LIMIT_ENABLED' - ORDER BY "posthog_instancesetting"."id" ASC - LIMIT 1 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ + SELECT COUNT(*) AS "__count" + FROM "posthog_annotation" + WHERE ((("posthog_annotation"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid + AND "posthog_annotation"."scope" = 'organization') + OR "posthog_annotation"."team_id" = 2) + AND NOT "posthog_annotation"."deleted") /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- # name: TestAnnotation.test_retrieving_annotation_is_not_n_plus_1.9 ''' - SELECT COUNT(*) AS "__count" + SELECT "posthog_annotation"."id", + "posthog_annotation"."content", + "posthog_annotation"."created_at", + "posthog_annotation"."updated_at", + "posthog_annotation"."dashboard_item_id", + "posthog_annotation"."team_id", + "posthog_annotation"."organization_id", + "posthog_annotation"."created_by_id", + "posthog_annotation"."scope", + "posthog_annotation"."creation_type", + "posthog_annotation"."date_marker", + "posthog_annotation"."deleted", + "posthog_annotation"."apply_all", + "posthog_dashboarditem"."id", + "posthog_dashboarditem"."name", + "posthog_dashboarditem"."derived_name", + "posthog_dashboarditem"."description", + "posthog_dashboarditem"."team_id", + "posthog_dashboarditem"."filters", + "posthog_dashboarditem"."filters_hash", + "posthog_dashboarditem"."query", + "posthog_dashboarditem"."order", + "posthog_dashboarditem"."deleted", + "posthog_dashboarditem"."saved", + "posthog_dashboarditem"."created_at", + "posthog_dashboarditem"."last_refresh", + "posthog_dashboarditem"."refreshing", + "posthog_dashboarditem"."created_by_id", + "posthog_dashboarditem"."is_sample", + "posthog_dashboarditem"."short_id", + "posthog_dashboarditem"."favorited", + "posthog_dashboarditem"."refresh_attempt", + "posthog_dashboarditem"."last_modified_at", + "posthog_dashboarditem"."last_modified_by_id", + "posthog_dashboarditem"."dashboard_id", + "posthog_dashboarditem"."layouts", + "posthog_dashboarditem"."color", + "posthog_dashboarditem"."dive_dashboard_id", + "posthog_dashboarditem"."updated_at", + "posthog_dashboarditem"."deprecated_tags", + "posthog_dashboarditem"."tags", + "posthog_user"."id", + "posthog_user"."password", + "posthog_user"."last_login", + "posthog_user"."first_name", + "posthog_user"."last_name", + "posthog_user"."is_staff", + "posthog_user"."is_active", + "posthog_user"."date_joined", + "posthog_user"."uuid", + "posthog_user"."current_organization_id", + "posthog_user"."current_team_id", + "posthog_user"."email", + "posthog_user"."pending_email", + "posthog_user"."temporary_token", + "posthog_user"."distinct_id", + "posthog_user"."is_email_verified", + "posthog_user"."requested_password_reset_at", + "posthog_user"."has_seen_product_intro_for", + "posthog_user"."strapi_id", + "posthog_user"."email_opt_in", + "posthog_user"."theme_mode", + "posthog_user"."partial_notification_settings", + "posthog_user"."anonymize_data", + "posthog_user"."toolbar_mode", + "posthog_user"."events_column_config" FROM "posthog_annotation" + LEFT OUTER JOIN "posthog_dashboarditem" ON ("posthog_annotation"."dashboard_item_id" = "posthog_dashboarditem"."id") + LEFT OUTER JOIN "posthog_user" ON ("posthog_annotation"."created_by_id" = "posthog_user"."id") WHERE ((("posthog_annotation"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid AND "posthog_annotation"."scope" = 'organization') OR "posthog_annotation"."team_id" = 2) - AND NOT "posthog_annotation"."deleted") /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ + AND NOT "posthog_annotation"."deleted") + ORDER BY "posthog_annotation"."date_marker" DESC + LIMIT 1000 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ ''' # --- From 4b02a6d1501b2eb0b0021005d588ef422388587a Mon Sep 17 00:00:00 2001 From: ted kaemming <65315+tkaemming@users.noreply.github.com> Date: Thu, 28 Mar 2024 10:50:50 -0700 Subject: [PATCH 7/7] Fix typo in comment Co-authored-by: Brett Hoerner --- posthog/settings/ingestion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/settings/ingestion.py b/posthog/settings/ingestion.py index 463db4121c440..8a3a24d981ce0 100644 --- a/posthog/settings/ingestion.py +++ b/posthog/settings/ingestion.py @@ -12,7 +12,7 @@ BUFFER_CONVERSION_SECONDS = get_from_env("BUFFER_CONVERSION_SECONDS", default=60, type_cast=int) # Whether or not random partitioning (i.e. overflow routing) should be allowed. -# (Enabling this setting this does not cause messages to be randomly +# (Enabling this setting does not cause messages to be randomly # partitioned.) Note that this setting, if disabled, takes precedence over other # partitioning-related settings below. CAPTURE_ALLOW_RANDOM_PARTITIONING = get_from_env("CAPTURE_ALLOW_RANDOM_PARTITIONING", True, type_cast=str_to_bool)