diff --git a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx index 2d8bba2f256bc..95a525987b8a2 100644 --- a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx +++ b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx @@ -6,6 +6,7 @@ import api from 'lib/api' import posthog from 'posthog-js' import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic' import { Scene } from 'scenes/sceneTypes' +import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' import { @@ -16,6 +17,7 @@ import { manualLinkSources, ManualLinkSourceType, PipelineTab, + ProductKey, SourceConfig, SourceFieldConfig, } from '~/types' @@ -731,6 +733,8 @@ export const sourceWizardLogic = kea([ ['resetTable', 'createTableSuccess'], dataWarehouseSettingsLogic, ['loadSources'], + teamLogic, + ['addProductIntent'], ], }), reducers({ @@ -1129,6 +1133,9 @@ export const sourceWizardLogic = kea([ setManualLinkingProvider: () => { actions.onNext() }, + selectConnector: () => { + actions.addProductIntent({ product_type: ProductKey.DATA_WAREHOUSE, intent_context: 'selected connector' }) + }, })), urlToAction(({ actions }) => ({ '/data-warehouse/:kind/redirect': ({ kind = '' }, searchParams) => { diff --git a/frontend/src/scenes/teamActivityDescriber.tsx b/frontend/src/scenes/teamActivityDescriber.tsx index 7c1a5cd22e44a..e3c4f9d27052f 100644 --- a/frontend/src/scenes/teamActivityDescriber.tsx +++ b/frontend/src/scenes/teamActivityDescriber.tsx @@ -347,6 +347,7 @@ const teamActionsMapping: Record< updated_at: () => null, uuid: () => null, live_events_token: () => null, + product_intents: () => null, } function nameAndLink(logItem?: ActivityLogItem): JSX.Element { diff --git a/frontend/src/scenes/teamLogic.tsx b/frontend/src/scenes/teamLogic.tsx index 3f6d102270525..0e6754609ca29 100644 --- a/frontend/src/scenes/teamLogic.tsx +++ b/frontend/src/scenes/teamLogic.tsx @@ -145,13 +145,14 @@ export const teamLogic = kea([ resetToken: async () => await api.update(`api/environments/${values.currentTeamId}/reset_token`, {}), addProductIntent: async ({ product_type, + intent_context, }: { product_type: ProductKey intent_context?: string | null }) => await api.update(`api/environments/${values.currentTeamId}/add_product_intent`, { product_type, - intent_context: null, + intent_context: intent_context ?? undefined, }), recordProductIntentOnboardingComplete: async ({ product_type }: { product_type: ProductKey }) => await api.update(`api/environments/${values.currentTeamId}/complete_product_onboarding`, { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 3d08ee1fb7512..6b78307b81774 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -545,6 +545,13 @@ export interface TeamType extends TeamBasicType { extra_settings?: Record modifiers?: HogQLQueryModifiers default_modifiers?: HogQLQueryModifiers + product_intents?: ProductIntentType[] +} + +export interface ProductIntentType { + product_type: string + created_at: string + onboarding_completed_at?: string } // This type would be more correct without `Partial`, but it's only used in the shared dashboard/insight diff --git a/plugin-server/src/config/kafka-topics.ts b/plugin-server/src/config/kafka-topics.ts index d7ea527904477..8610bf8f0b819 100644 --- a/plugin-server/src/config/kafka-topics.ts +++ b/plugin-server/src/config/kafka-topics.ts @@ -38,8 +38,6 @@ export const KAFKA_CLICKHOUSE_SESSION_REPLAY_EVENTS = `${prefix}clickhouse_sessi export const KAFKA_PERFORMANCE_EVENTS = `${prefix}clickhouse_performance_events${suffix}` // write heatmap events to ClickHouse export const KAFKA_CLICKHOUSE_HEATMAP_EVENTS = `${prefix}clickhouse_heatmap_events${suffix}` -// write exception events to ClickHouse -export const KAFKA_EXCEPTION_SYMBOLIFICATION_EVENTS = `${prefix}exception_symbolification_events${suffix}` // log entries for ingestion into ClickHouse export const KAFKA_LOG_ENTRIES = `${prefix}log_entries${suffix}` @@ -47,3 +45,7 @@ export const KAFKA_LOG_ENTRIES = `${prefix}log_entries${suffix}` // CDP topics export const KAFKA_CDP_FUNCTION_CALLBACKS = `${prefix}cdp_function_callbacks${suffix}` export const KAFKA_CDP_FUNCTION_OVERFLOW = `${prefix}cdp_function_overflow${suffix}` + +// Error tracking topics +export const KAFKA_EXCEPTION_SYMBOLIFICATION_EVENTS = `${prefix}exception_symbolification_events${suffix}` +export const KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT = `${prefix}clickhouse_error_tracking_issue_fingerprint${suffix}` diff --git a/posthog/api/project.py b/posthog/api/project.py index 0c60c7e649a91..8efca11b21968 100644 --- a/posthog/api/project.py +++ b/posthog/api/project.py @@ -44,7 +44,11 @@ TeamMemberStrictManagementPermission, ) from posthog.user_permissions import UserPermissions, UserPermissionsSerializerMixin -from posthog.utils import get_ip_address, get_week_start_for_country_code +from posthog.utils import ( + get_instance_realm, + get_ip_address, + get_week_start_for_country_code, +) class ProjectSerializer(serializers.ModelSerializer): @@ -195,7 +199,9 @@ def get_live_events_token(self, project: Project) -> Optional[str]: def get_product_intents(self, obj): project = obj team = project.passthrough_team - return ProductIntent.objects.filter(team=team).values("product_type", "created_at", "onboarding_completed_at") + return ProductIntent.objects.filter(team=team).values( + "product_type", "created_at", "onboarding_completed_at", "updated_at" + ) @staticmethod def validate_session_recording_linked_flag(value) -> dict | None: @@ -572,7 +578,7 @@ def add_product_intent(self, request: request.Request, *args, **kwargs): product_intent.updated_at = datetime.now(tz=UTC) product_intent.save() - if created and isinstance(user, User): + if isinstance(user, User): report_user_action( user, "user showed product intent", @@ -582,6 +588,10 @@ def add_product_intent(self, request: request.Request, *args, **kwargs): "$current_url": current_url, "$session_id": session_id, "intent_context": request.data.get("intent_context"), + "is_first_intent_for_product": created, + "intent_created_at": product_intent.created_at, + "intent_updated_at": product_intent.updated_at, + "realm": get_instance_realm(), }, team=team, ) @@ -612,6 +622,10 @@ def complete_product_onboarding(self, request: request.Request, *args, **kwargs) "$current_url": current_url, "$session_id": session_id, "intent_context": request.data.get("intent_context"), + "is_first_intent_for_product": created, + "intent_created_at": product_intent.created_at, + "intent_updated_at": product_intent.updated_at, + "realm": get_instance_realm(), }, team=team, ) @@ -626,6 +640,10 @@ def complete_product_onboarding(self, request: request.Request, *args, **kwargs) "product_key": product_type, "$current_url": current_url, "$session_id": session_id, + "intent_context": request.data.get("intent_context"), + "intent_created_at": product_intent.created_at, + "intent_updated_at": product_intent.updated_at, + "realm": get_instance_realm(), }, team=team, ) diff --git a/posthog/api/team.py b/posthog/api/team.py index b92f3acaf35ae..148471919cbc0 100644 --- a/posthog/api/team.py +++ b/posthog/api/team.py @@ -6,17 +6,17 @@ from django.shortcuts import get_object_or_404 from loginas.utils import is_impersonated_session -from posthog.auth import PersonalAPIKeyAuthentication -from posthog.jwt import PosthogJwtAudience, encode_jwt from rest_framework import exceptions, request, response, serializers, viewsets from rest_framework.permissions import BasePermission, IsAuthenticated from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import TeamBasicSerializer from posthog.api.utils import action +from posthog.auth import PersonalAPIKeyAuthentication from posthog.constants import AvailableFeature from posthog.event_usage import report_user_action from posthog.geoip import get_geoip_properties +from posthog.jwt import PosthogJwtAudience, encode_jwt from posthog.models import ProductIntent, Team, User from posthog.models.activity_logging.activity_log import ( Detail, @@ -43,7 +43,11 @@ get_organization_from_view, ) from posthog.user_permissions import UserPermissions, UserPermissionsSerializerMixin -from posthog.utils import get_ip_address, get_week_start_for_country_code +from posthog.utils import ( + get_instance_realm, + get_ip_address, + get_week_start_for_country_code, +) class PremiumMultiProjectPermissions(BasePermission): # TODO: Rename to include "Env" in name @@ -211,7 +215,9 @@ def get_live_events_token(self, team: Team) -> Optional[str]: ) def get_product_intents(self, obj): - return ProductIntent.objects.filter(team=obj).values("product_type", "created_at", "onboarding_completed_at") + return ProductIntent.objects.filter(team=obj).values( + "product_type", "created_at", "onboarding_completed_at", "updated_at" + ) @staticmethod def validate_session_recording_linked_flag(value) -> dict | None: @@ -582,7 +588,7 @@ def add_product_intent(self, request: request.Request, *args, **kwargs): product_intent.updated_at = datetime.now(tz=UTC) product_intent.save() - if created and isinstance(user, User): + if isinstance(user, User): report_user_action( user, "user showed product intent", @@ -592,6 +598,10 @@ def add_product_intent(self, request: request.Request, *args, **kwargs): "$current_url": current_url, "$session_id": session_id, "intent_context": request.data.get("intent_context"), + "is_first_intent_for_product": created, + "intent_created_at": product_intent.created_at, + "intent_updated_at": product_intent.updated_at, + "realm": get_instance_realm(), }, team=team, ) @@ -621,6 +631,10 @@ def complete_product_onboarding(self, request: request.Request, *args, **kwargs) "$current_url": current_url, "$session_id": session_id, "intent_context": request.data.get("intent_context"), + "is_first_intent_for_product": created, + "intent_created_at": product_intent.created_at, + "intent_updated_at": product_intent.updated_at, + "realm": get_instance_realm(), }, team=team, ) @@ -635,6 +649,10 @@ def complete_product_onboarding(self, request: request.Request, *args, **kwargs) "product_key": product_type, "$current_url": current_url, "$session_id": session_id, + "intent_context": request.data.get("intent_context"), + "intent_created_at": product_intent.created_at, + "intent_updated_at": product_intent.updated_at, + "realm": get_instance_realm(), }, team=team, ) diff --git a/posthog/api/test/__snapshots__/test_api_docs.ambr b/posthog/api/test/__snapshots__/test_api_docs.ambr index a5f9b394809ae..6ef31c6530176 100644 --- a/posthog/api/test/__snapshots__/test_api_docs.ambr +++ b/posthog/api/test/__snapshots__/test_api_docs.ambr @@ -97,8 +97,8 @@ '/home/runner/work/posthog/posthog/posthog/api/survey.py: Warning [SurveyViewSet > SurveySerializer]: unable to resolve type hint for function "get_conditions". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/web_experiment.py: Warning [WebExperimentViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.web_experiment.WebExperiment" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', 'Warning: encountered multiple names for the same choice set (HrefMatchingEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', - 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "KindCfaEnum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "Kind069Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', + 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "KindCfaEnum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "type". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "TypeF73Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: encountered multiple names for the same choice set (EffectivePrivilegeLevelEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: encountered multiple names for the same choice set (MembershipLevelEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', diff --git a/posthog/api/test/__snapshots__/test_decide.ambr b/posthog/api/test/__snapshots__/test_decide.ambr index 93ec860efb05b..18fc2e400e3fd 100644 --- a/posthog/api/test/__snapshots__/test_decide.ambr +++ b/posthog/api/test/__snapshots__/test_decide.ambr @@ -523,7 +523,8 @@ ''' SELECT "posthog_productintent"."product_type", "posthog_productintent"."created_at", - "posthog_productintent"."onboarding_completed_at" + "posthog_productintent"."onboarding_completed_at", + "posthog_productintent"."updated_at" FROM "posthog_productintent" WHERE "posthog_productintent"."team_id" = 2 ''' diff --git a/posthog/api/test/test_team.py b/posthog/api/test/test_team.py index ba697d073699f..3a881facfea44 100644 --- a/posthog/api/test/test_team.py +++ b/posthog/api/test/test_team.py @@ -26,6 +26,7 @@ from posthog.temporal.common.client import sync_connect from posthog.temporal.common.schedule import describe_schedule from posthog.test.base import APIBaseTest +from posthog.utils import get_instance_realm def team_api_test_factory(): @@ -1042,6 +1043,10 @@ def test_can_add_product_intent( "$session_id": "test_session_id", "intent_context": "onboarding product selected", "$set_once": {"first_onboarding_product_selected": "product_analytics"}, + "is_first_intent_for_product": True, + "intent_created_at": datetime(2024, 1, 1, 0, 0, 0, tzinfo=UTC), + "intent_updated_at": datetime(2024, 1, 1, 0, 0, 0, tzinfo=UTC), + "realm": get_instance_realm(), }, team=self.team, ) @@ -1073,6 +1078,10 @@ def test_can_complete_product_onboarding( "product_key": "product_analytics", "$current_url": "https://posthogtest.com/my-url", "$session_id": "test_session_id", + "intent_context": None, + "intent_created_at": datetime(2024, 1, 1, 0, 0, 0, tzinfo=UTC), + "intent_updated_at": datetime(2024, 1, 5, 0, 0, 0, tzinfo=UTC), + "realm": get_instance_realm(), }, team=self.team, ) diff --git a/posthog/clickhouse/migrations/0083_add_error_tracking_issue_fingerprint_overrides_table_and_consumer.py b/posthog/clickhouse/migrations/0083_add_error_tracking_issue_fingerprint_overrides_table_and_consumer.py new file mode 100644 index 0000000000000..1d24e713ee6c3 --- /dev/null +++ b/posthog/clickhouse/migrations/0083_add_error_tracking_issue_fingerprint_overrides_table_and_consumer.py @@ -0,0 +1,12 @@ +from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions +from posthog.models.error_tracking.sql import ( + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL, + KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL, + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_MV_SQL, +) + +operations = [ + run_sql_with_exceptions(ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL()), + run_sql_with_exceptions(KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL()), + run_sql_with_exceptions(ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_MV_SQL), +] diff --git a/posthog/clickhouse/schema.py b/posthog/clickhouse/schema.py index e3f5044b7a419..1df1fa52792af 100644 --- a/posthog/clickhouse/schema.py +++ b/posthog/clickhouse/schema.py @@ -84,6 +84,11 @@ PERSON_DISTINCT_ID_OVERRIDES_MV_SQL, KAFKA_PERSON_DISTINCT_ID_OVERRIDES_TABLE_SQL, ) +from posthog.models.error_tracking.sql import ( + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL, + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_MV_SQL, + KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL, +) from posthog.models.person_overrides.sql import ( PERSON_OVERRIDES_CREATE_TABLE_SQL, PERSON_OVERRIDES_CREATE_DICTIONARY_SQL, @@ -130,6 +135,7 @@ PERSONS_DISTINCT_ID_TABLE_SQL, PERSON_DISTINCT_ID2_TABLE_SQL, PERSON_DISTINCT_ID_OVERRIDES_TABLE_SQL, + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL, PLUGIN_LOG_ENTRIES_TABLE_SQL, SESSION_RECORDING_EVENTS_TABLE_SQL, INGESTION_WARNINGS_DATA_TABLE_SQL, @@ -170,6 +176,7 @@ KAFKA_PERSONS_DISTINCT_ID_TABLE_SQL, KAFKA_PERSON_DISTINCT_ID2_TABLE_SQL, KAFKA_PERSON_DISTINCT_ID_OVERRIDES_TABLE_SQL, + KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL, KAFKA_PLUGIN_LOG_ENTRIES_TABLE_SQL, KAFKA_SESSION_RECORDING_EVENTS_TABLE_SQL, KAFKA_INGESTION_WARNINGS_TABLE_SQL, @@ -189,6 +196,7 @@ PERSONS_DISTINCT_ID_TABLE_MV_SQL, PERSON_DISTINCT_ID2_MV_SQL, PERSON_DISTINCT_ID_OVERRIDES_MV_SQL, + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_MV_SQL, PLUGIN_LOG_ENTRIES_TABLE_MV_SQL, SESSION_RECORDING_EVENTS_TABLE_MV_SQL, INGESTION_WARNINGS_MV_TABLE_SQL, diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr index ac13d60ef2d66..bc5a623ed0cdc 100644 --- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr +++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr @@ -74,6 +74,21 @@ ''' # --- +# name: test_create_kafka_table_with_different_kafka_host[kafka_error_tracking_issue_fingerprint_overrides] + ''' + + CREATE TABLE IF NOT EXISTS kafka_error_tracking_issue_fingerprint_overrides ON CLUSTER 'posthog' + ( + team_id Int64, + fingerprint VARCHAR, + issue_id UUID, + is_deleted Int8, + version Int64 + + ) ENGINE = Kafka('test.kafka.broker:9092', 'clickhouse_error_tracking_issue_fingerprint_test', 'clickhouse-error-tracking-issue-fingerprint-overrides', 'JSONEachRow') + + ''' +# --- # name: test_create_kafka_table_with_different_kafka_host[kafka_events_dead_letter_queue] ''' @@ -558,6 +573,50 @@ Order By (team_id, cohort_id, person_id, version) + ''' +# --- +# name: test_create_table_query[error_tracking_issue_fingerprint_overrides] + ''' + + CREATE TABLE IF NOT EXISTS error_tracking_issue_fingerprint_overrides ON CLUSTER 'posthog' + ( + team_id Int64, + fingerprint VARCHAR, + issue_id UUID, + is_deleted Int8, + version Int64 + + + , _timestamp DateTime + , _offset UInt64 + , _partition UInt64 + + , INDEX kafka_timestamp_minmax_error_tracking_issue_fingerprint_overrides _timestamp TYPE minmax GRANULARITY 3 + + ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.error_tracking_issue_fingerprint_overrides', '{replica}-{shard}', version) + + ORDER BY (team_id, fingerprint) + SETTINGS index_granularity = 512 + + ''' +# --- +# name: test_create_table_query[error_tracking_issue_fingerprint_overrides_mv] + ''' + + CREATE MATERIALIZED VIEW IF NOT EXISTS error_tracking_issue_fingerprint_overrides_mv ON CLUSTER 'posthog' + TO posthog_test.error_tracking_issue_fingerprint_overrides + AS SELECT + team_id, + fingerprint, + issue_id, + is_deleted, + version, + _timestamp, + _offset, + _partition + FROM posthog_test.kafka_error_tracking_issue_fingerprint_overrides + WHERE version > 0 -- only store updated rows, not newly inserted ones + ''' # --- # name: test_create_table_query[events] @@ -877,6 +936,21 @@ ''' # --- +# name: test_create_table_query[kafka_error_tracking_issue_fingerprint_overrides] + ''' + + CREATE TABLE IF NOT EXISTS kafka_error_tracking_issue_fingerprint_overrides ON CLUSTER 'posthog' + ( + team_id Int64, + fingerprint VARCHAR, + issue_id UUID, + is_deleted Int8, + version Int64 + + ) ENGINE = Kafka('kafka:9092', 'clickhouse_error_tracking_issue_fingerprint_test', 'clickhouse-error-tracking-issue-fingerprint-overrides', 'JSONEachRow') + + ''' +# --- # name: test_create_table_query[kafka_events_dead_letter_queue] ''' @@ -2910,6 +2984,31 @@ Order By (team_id, cohort_id, person_id, version) + ''' +# --- +# name: test_create_table_query_replicated_and_storage[error_tracking_issue_fingerprint_overrides] + ''' + + CREATE TABLE IF NOT EXISTS error_tracking_issue_fingerprint_overrides ON CLUSTER 'posthog' + ( + team_id Int64, + fingerprint VARCHAR, + issue_id UUID, + is_deleted Int8, + version Int64 + + + , _timestamp DateTime + , _offset UInt64 + , _partition UInt64 + + , INDEX kafka_timestamp_minmax_error_tracking_issue_fingerprint_overrides _timestamp TYPE minmax GRANULARITY 3 + + ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.error_tracking_issue_fingerprint_overrides', '{replica}-{shard}', version) + + ORDER BY (team_id, fingerprint) + SETTINGS index_granularity = 512 + ''' # --- # name: test_create_table_query_replicated_and_storage[events_dead_letter_queue] diff --git a/posthog/conftest.py b/posthog/conftest.py index eb7c5500adb19..938ae3028bfeb 100644 --- a/posthog/conftest.py +++ b/posthog/conftest.py @@ -68,6 +68,7 @@ def reset_clickhouse_tables(): TRUNCATE_PERSON_STATIC_COHORT_TABLE_SQL, TRUNCATE_PERSON_TABLE_SQL, ) + from posthog.models.error_tracking.sql import TRUNCATE_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL from posthog.models.sessions.sql import TRUNCATE_SESSIONS_TABLE_SQL from posthog.session_recordings.sql.session_recording_event_sql import ( TRUNCATE_SESSION_RECORDING_EVENTS_TABLE_SQL, @@ -81,6 +82,7 @@ def reset_clickhouse_tables(): TRUNCATE_PERSON_DISTINCT_ID2_TABLE_SQL, TRUNCATE_PERSON_DISTINCT_ID_OVERRIDES_TABLE_SQL, TRUNCATE_PERSON_STATIC_COHORT_TABLE_SQL, + TRUNCATE_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL, TRUNCATE_SESSION_RECORDING_EVENTS_TABLE_SQL(), TRUNCATE_PLUGIN_LOG_ENTRIES_TABLE_SQL, TRUNCATE_COHORTPEOPLE_TABLE_SQL, diff --git a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr index 06e46e0ca30b0..f018e96ef067a 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr @@ -31,7 +31,7 @@ FROM events LEFT JOIN ( SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [6]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) + WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [4]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) WHERE and(equals(events.team_id, 420), 1, ifNull(equals(__in_cohort.matched, 1), 0)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0 @@ -42,7 +42,7 @@ FROM events LEFT JOIN ( SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id FROM static_cohort_people - WHERE in(cohort_id, [6])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) + WHERE in(cohort_id, [4])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) WHERE and(1, equals(__in_cohort.matched, 1)) LIMIT 100 ''' @@ -55,7 +55,7 @@ FROM events LEFT JOIN ( SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [7]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) + WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [5]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) WHERE and(equals(events.team_id, 420), 1, ifNull(equals(__in_cohort.matched, 1), 0)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0 @@ -66,7 +66,7 @@ FROM events LEFT JOIN ( SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id FROM static_cohort_people - WHERE in(cohort_id, [7])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) + WHERE in(cohort_id, [5])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) WHERE and(1, equals(__in_cohort.matched, 1)) LIMIT 100 ''' diff --git a/posthog/kafka_client/topics.py b/posthog/kafka_client/topics.py index 227ca18d666fc..fa58d40c5fa36 100644 --- a/posthog/kafka_client/topics.py +++ b/posthog/kafka_client/topics.py @@ -23,8 +23,6 @@ KAFKA_CLICKHOUSE_HEATMAP_EVENTS = f"{KAFKA_PREFIX}clickhouse_heatmap_events{SUFFIX}" -KAFKA_EXCEPTION_SYMBOLIFICATION_EVENTS = f"{KAFKA_PREFIX}exception_symbolification_events{SUFFIX}" - # from capture to recordings consumer KAFKA_SESSION_RECORDING_EVENTS = f"{KAFKA_PREFIX}session_recording_events{SUFFIX}" # from capture to recordings blob ingestion consumer @@ -34,3 +32,6 @@ # from recordings consumer to clickhouse KAFKA_CLICKHOUSE_SESSION_REPLAY_EVENTS = f"{KAFKA_PREFIX}clickhouse_session_replay_events{SUFFIX}" KAFKA_CLICKHOUSE_SESSION_RECORDING_EVENTS = f"{KAFKA_PREFIX}clickhouse_session_recording_events{SUFFIX}" + +KAFKA_EXCEPTION_SYMBOLIFICATION_EVENTS = f"{KAFKA_PREFIX}exception_symbolification_events{SUFFIX}" +KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT = f"{KAFKA_PREFIX}clickhouse_error_tracking_issue_fingerprint{SUFFIX}" diff --git a/posthog/models/error_tracking/sql.py b/posthog/models/error_tracking/sql.py new file mode 100644 index 0000000000000..1700a6c7c4cd9 --- /dev/null +++ b/posthog/models/error_tracking/sql.py @@ -0,0 +1,79 @@ +from posthog.clickhouse.indexes import index_by_kafka_timestamp +from posthog.clickhouse.kafka_engine import KAFKA_COLUMNS_WITH_PARTITION, kafka_engine +from posthog.clickhouse.table_engines import ReplacingMergeTree +from posthog.kafka_client.topics import KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT +from posthog.settings import CLICKHOUSE_CLUSTER, CLICKHOUSE_DATABASE + +# +# error_tracking_issue_fingerprint_overrides: This table contains rows for all (team_id, fingerprint) +# pairs where the $exception_issue_id has changed. +# + +ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE = "error_tracking_issue_fingerprint_overrides" + +ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_BASE_SQL = """ +CREATE TABLE IF NOT EXISTS {table_name} ON CLUSTER '{cluster}' +( + team_id Int64, + fingerprint VARCHAR, + issue_id UUID, + is_deleted Int8, + version Int64 + {extra_fields} +) ENGINE = {engine} +""" + +ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_ENGINE = lambda: ReplacingMergeTree( + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE, ver="version" +) + +ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL = lambda: ( + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_BASE_SQL + + """ + ORDER BY (team_id, fingerprint) + SETTINGS index_granularity = 512 + """ +).format( + table_name=ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE, + cluster=CLICKHOUSE_CLUSTER, + engine=ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_ENGINE(), + extra_fields=f""" + {KAFKA_COLUMNS_WITH_PARTITION} + , {index_by_kafka_timestamp(ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE)} + """, +) + +KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL = ( + lambda: ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_BASE_SQL.format( + table_name="kafka_" + ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE, + cluster=CLICKHOUSE_CLUSTER, + engine=kafka_engine( + KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT, group="clickhouse-error-tracking-issue-fingerprint-overrides" + ), + extra_fields="", + ) +) + +ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_MV_SQL = """ +CREATE MATERIALIZED VIEW IF NOT EXISTS {table_name}_mv ON CLUSTER '{cluster}' +TO {database}.{table_name} +AS SELECT +team_id, +fingerprint, +issue_id, +is_deleted, +version, +_timestamp, +_offset, +_partition +FROM {database}.kafka_{table_name} +WHERE version > 0 -- only store updated rows, not newly inserted ones +""".format( + table_name=ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE, + cluster=CLICKHOUSE_CLUSTER, + database=CLICKHOUSE_DATABASE, +) + +TRUNCATE_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL = ( + f"TRUNCATE TABLE IF EXISTS {ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE} ON CLUSTER '{CLICKHOUSE_CLUSTER}'" +)