diff --git a/plugin-server/src/worker/ingestion/event-pipeline/createEventStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/createEventStep.ts index ea717cfa565dd..f04ed8d3b42c9 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/createEventStep.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/createEventStep.ts @@ -1,4 +1,4 @@ -import { Person, PreIngestionEvent, RawClickHouseEvent } from '../../../types' +import { Person, PreIngestionEvent, RawKafkaEvent } from '../../../types' import { EventPipelineRunner } from './runner' export function createEventStep( @@ -6,6 +6,6 @@ export function createEventStep( event: PreIngestionEvent, person: Person, processPerson: boolean -): [RawClickHouseEvent, Promise] { +): RawKafkaEvent { return runner.eventsProcessor.createEvent(event, person, processPerson) } diff --git a/plugin-server/src/worker/ingestion/event-pipeline/emitEventStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/emitEventStep.ts new file mode 100644 index 0000000000000..5043128e40538 --- /dev/null +++ b/plugin-server/src/worker/ingestion/event-pipeline/emitEventStep.ts @@ -0,0 +1,6 @@ +import { RawKafkaEvent } from '../../../types' +import { EventPipelineRunner } from './runner' + +export function emitEventStep(runner: EventPipelineRunner, event: RawKafkaEvent): [Promise] { + return [runner.eventsProcessor.emitEvent(event)] +} diff --git a/plugin-server/src/worker/ingestion/event-pipeline/produceExceptionSymbolificationEventStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/produceExceptionSymbolificationEventStep.ts index 91173639a3c0f..d0f3712d8aeb0 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/produceExceptionSymbolificationEventStep.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/produceExceptionSymbolificationEventStep.ts @@ -1,10 +1,10 @@ -import { RawClickHouseEvent } from '../../../types' +import { RawKafkaEvent } from '../../../types' import { status } from '../../../utils/status' import { EventPipelineRunner } from './runner' export function produceExceptionSymbolificationEventStep( runner: EventPipelineRunner, - event: RawClickHouseEvent + event: RawKafkaEvent ): Promise<[Promise]> { const ack = runner.hub.kafkaProducer .produce({ diff --git a/plugin-server/src/worker/ingestion/event-pipeline/runner.ts b/plugin-server/src/worker/ingestion/event-pipeline/runner.ts index f2df42e0cecae..157c5fc252d4d 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/runner.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/runner.ts @@ -11,6 +11,7 @@ import { status } from '../../../utils/status' import { EventsProcessor } from '../process-event' import { captureIngestionWarning, generateEventDeadLetterQueueMessage } from '../utils' import { createEventStep } from './createEventStep' +import { emitEventStep } from './emitEventStep' import { enrichExceptionEventStep } from './enrichExceptionEventStep' import { extractHeatmapDataStep } from './extractHeatmapDataStep' import { @@ -259,24 +260,25 @@ export class EventPipelineRunner { event.team_id ) - const [rawClickhouseEvent, eventAck] = await this.runStep( + const rawEvent = await this.runStep( createEventStep, [this, enrichedIfErrorEvent, person, processPerson], event.team_id ) - kafkaAcks.push(eventAck) if (event.event === '$exception' && event.team_id == 2) { const [exceptionAck] = await this.runStep( produceExceptionSymbolificationEventStep, - [this, rawClickhouseEvent], + [this, rawEvent], event.team_id ) kafkaAcks.push(exceptionAck) - return this.registerLastStep('produceExceptionSymbolificationEventStep', [rawClickhouseEvent], kafkaAcks) + return this.registerLastStep('produceExceptionSymbolificationEventStep', [rawEvent], kafkaAcks) + } else { + const [clickhouseAck] = await this.runStep(emitEventStep, [this, rawEvent], event.team_id) + kafkaAcks.push(clickhouseAck) + return this.registerLastStep('emitEventStep', [rawEvent], kafkaAcks) } - - return this.registerLastStep('createEventStep', [rawClickhouseEvent], kafkaAcks) } registerLastStep(stepName: string, args: any[], ackPromises?: Array>): EventPipelineResult { diff --git a/plugin-server/src/worker/ingestion/process-event.ts b/plugin-server/src/worker/ingestion/process-event.ts index 4c38a68d085ec..2162d243d3b77 100644 --- a/plugin-server/src/worker/ingestion/process-event.ts +++ b/plugin-server/src/worker/ingestion/process-event.ts @@ -204,11 +204,7 @@ export class EventsProcessor { return res } - createEvent( - preIngestionEvent: PreIngestionEvent, - person: Person, - processPerson: boolean - ): [RawKafkaEvent, Promise] { + createEvent(preIngestionEvent: PreIngestionEvent, person: Person, processPerson: boolean): RawKafkaEvent { const { eventUuid: uuid, event, teamId, projectId, distinctId, properties, timestamp } = preIngestionEvent let elementsChain = '' @@ -264,10 +260,14 @@ export class EventsProcessor { person_mode: personMode, } + return rawEvent + } + + emitEvent(rawEvent: RawKafkaEvent): Promise { const ack = this.kafkaProducer .produce({ topic: this.pluginsServer.CLICKHOUSE_JSON_EVENTS_KAFKA_TOPIC, - key: uuid, + key: rawEvent.uuid, value: Buffer.from(JSON.stringify(rawEvent)), waitForAck: true, }) @@ -275,16 +275,16 @@ export class EventsProcessor { // Some messages end up significantly larger than the original // after plugin processing, person & group enrichment, etc. if (error instanceof MessageSizeTooLarge) { - await captureIngestionWarning(this.db.kafkaProducer, teamId, 'message_size_too_large', { - eventUuid: uuid, - distinctId: distinctId, + await captureIngestionWarning(this.db.kafkaProducer, rawEvent.team_id, 'message_size_too_large', { + eventUuid: rawEvent.uuid, + distinctId: rawEvent.distinct_id, }) } else { throw error } }) - return [rawEvent, ack] + return ack } private async upsertGroup( diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/runner.test.ts.snap b/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/runner.test.ts.snap index 1d1b12faa29ac..37a9ace98cd74 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/runner.test.ts.snap +++ b/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/runner.test.ts.snap @@ -144,5 +144,24 @@ Array [ true, ], ], + Array [ + "emitEventStep", + Array [ + Object { + "created_at": "2024-11-18 14:54:33.606", + "distinct_id": "my_id", + "elements_chain": "", + "event": "$pageview", + "person_created_at": "2024-11-18 14:54:33", + "person_mode": "full", + "person_properties": "{}", + "project_id": 1, + "properties": "{}", + "team_id": 2, + "timestamp": "2020-02-23 02:15:00.000", + "uuid": "uuid1", + }, + ], + ], ] `; diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts index 80b59cc606c3d..692f5af76efb0 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts @@ -120,7 +120,7 @@ describe('prepareEventStep()', () => { it('extracts elements_chain from properties', async () => { const event: PluginEvent = { ...pluginEvent, ip: null, properties: { $elements_chain: 'random string', a: 1 } } const preppedEvent = await prepareEventStep(runner, event) - const [chEvent, _] = runner.eventsProcessor.createEvent(preppedEvent, person) + const chEvent = runner.eventsProcessor.createEvent(preppedEvent, person) expect(chEvent.elements_chain).toEqual('random string') expect(chEvent.properties).toEqual('{"a":1}') @@ -137,7 +137,7 @@ describe('prepareEventStep()', () => { }, } const preppedEvent = await prepareEventStep(runner, event) - const [chEvent, _] = runner.eventsProcessor.createEvent(preppedEvent, person) + const chEvent = runner.eventsProcessor.createEvent(preppedEvent, person) expect(chEvent.elements_chain).toEqual('random string') expect(chEvent.properties).toEqual('{"a":1}') @@ -151,7 +151,7 @@ describe('prepareEventStep()', () => { properties: { a: 1, $elements: [{ tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: 'text' }] }, } const preppedEvent = await prepareEventStep(runner, event) - const [chEvent, _] = runner.eventsProcessor.createEvent(preppedEvent, person) + const chEvent = runner.eventsProcessor.createEvent(preppedEvent, person) expect(chEvent.elements_chain).toEqual('div:nth-child="1"nth-of-type="2"text="text"') expect(chEvent.properties).toEqual('{"a":1}') diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts index 58ffa94278015..9a8576e540982 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts @@ -1,9 +1,10 @@ import { PluginEvent } from '@posthog/plugin-scaffold' import { DateTime } from 'luxon' -import { ISOTimestamp, Person, PipelineEvent, PreIngestionEvent } from '../../../../src/types' +import { ISOTimestamp, Person, PipelineEvent, PreIngestionEvent, RawKafkaEvent } from '../../../../src/types' import { createEventsToDropByToken } from '../../../../src/utils/db/hub' import { createEventStep } from '../../../../src/worker/ingestion/event-pipeline/createEventStep' +import { emitEventStep } from '../../../../src/worker/ingestion/event-pipeline/emitEventStep' import * as metrics from '../../../../src/worker/ingestion/event-pipeline/metrics' import { pluginsProcessEventStep } from '../../../../src/worker/ingestion/event-pipeline/pluginsProcessEventStep' import { populateTeamDataStep } from '../../../../src/worker/ingestion/event-pipeline/populateTeamDataStep' @@ -18,6 +19,7 @@ jest.mock('../../../../src/worker/ingestion/event-pipeline/pluginsProcessEventSt jest.mock('../../../../src/worker/ingestion/event-pipeline/processPersonsStep') jest.mock('../../../../src/worker/ingestion/event-pipeline/prepareEventStep') jest.mock('../../../../src/worker/ingestion/event-pipeline/createEventStep') +jest.mock('../../../../src/worker/ingestion/event-pipeline/emitEventStep') jest.mock('../../../../src/worker/ingestion/event-pipeline/runAsyncHandlersStep') class TestEventPipelineRunner extends EventPipelineRunner { @@ -74,6 +76,21 @@ const preIngestionEvent: PreIngestionEvent = { elementsList: [], } +const createdEvent: RawKafkaEvent = { + created_at: '2024-11-18 14:54:33.606', + distinct_id: 'my_id', + elements_chain: '', + event: '$pageview', + person_created_at: '2024-11-18 14:54:33', + person_mode: 'full', + person_properties: '{}', + project_id: 1, + properties: '{}', + team_id: 2, + timestamp: '2020-02-23 02:15:00.000', + uuid: 'uuid1', +} + const person: Person = { id: 123, team_id: 2, @@ -112,7 +129,8 @@ describe('EventPipelineRunner', () => { { person, personUpdateProperties: {}, get: () => Promise.resolve(person) } as any, ]) jest.mocked(prepareEventStep).mockResolvedValue(preIngestionEvent) - jest.mocked(createEventStep).mockResolvedValue([null, Promise.resolve()]) + jest.mocked(createEventStep).mockResolvedValue(createdEvent) + jest.mocked(emitEventStep).mockResolvedValue([Promise.resolve()]) jest.mocked(processOnEventStep).mockResolvedValue(null) }) @@ -129,6 +147,7 @@ describe('EventPipelineRunner', () => { 'extractHeatmapDataStep', 'enrichExceptionEventStep', 'createEventStep', + 'emitEventStep', ]) expect(runner.stepsWithArgs).toMatchSnapshot() }) @@ -158,6 +177,7 @@ describe('EventPipelineRunner', () => { 'extractHeatmapDataStep', 'enrichExceptionEventStep', 'createEventStep', + 'emitEventStep', ]) }) @@ -179,11 +199,11 @@ describe('EventPipelineRunner', () => { const result = await runner.runEventPipeline(pipelineEvent) expect(result.error).toBeUndefined() - expect(pipelineStepMsSummarySpy).toHaveBeenCalledTimes(8) + expect(pipelineStepMsSummarySpy).toHaveBeenCalledTimes(9) expect(pipelineLastStepCounterSpy).toHaveBeenCalledTimes(1) expect(eventProcessedAndIngestedCounterSpy).toHaveBeenCalledTimes(1) - expect(pipelineStepMsSummarySpy).toHaveBeenCalledWith('createEventStep') - expect(pipelineLastStepCounterSpy).toHaveBeenCalledWith('createEventStep') + expect(pipelineStepMsSummarySpy).toHaveBeenCalledWith('emitEventStep') + expect(pipelineLastStepCounterSpy).toHaveBeenCalledWith('emitEventStep') expect(pipelineStepErrorCounterSpy).not.toHaveBeenCalled() }) @@ -380,6 +400,7 @@ describe('EventPipelineRunner', () => { 'extractHeatmapDataStep', 'enrichExceptionEventStep', 'createEventStep', + 'emitEventStep', ]) }) }) diff --git a/plugin-server/tests/worker/ingestion/process-event.test.ts b/plugin-server/tests/worker/ingestion/process-event.test.ts index 4e828ebd8ca62..6886e7723ec05 100644 --- a/plugin-server/tests/worker/ingestion/process-event.test.ts +++ b/plugin-server/tests/worker/ingestion/process-event.test.ts @@ -89,7 +89,7 @@ describe('EventsProcessor#createEvent()', () => { it('emits event with person columns, re-using event properties', async () => { const processPerson = true - eventsProcessor.createEvent(preIngestionEvent, person, processPerson) + eventsProcessor.emitEvent(eventsProcessor.createEvent(preIngestionEvent, person, processPerson)) await eventsProcessor.kafkaProducer.flush() @@ -147,10 +147,12 @@ describe('EventsProcessor#createEvent()', () => { ) const processPerson = true - eventsProcessor.createEvent( - { ...preIngestionEvent, properties: { $group_0: 'group_key' } }, - person, - processPerson + eventsProcessor.emitEvent( + eventsProcessor.createEvent( + { ...preIngestionEvent, properties: { $group_0: 'group_key' } }, + person, + processPerson + ) ) const events = await delayUntilEventIngested(() => hub.db.fetchEvents()) @@ -169,10 +171,12 @@ describe('EventsProcessor#createEvent()', () => { it('when $process_person_profile=false, emits event with without person properties or groups', async () => { const processPerson = false - eventsProcessor.createEvent( - { ...preIngestionEvent, properties: { $group_0: 'group_key' } }, - person, - processPerson + eventsProcessor.emitEvent( + eventsProcessor.createEvent( + { ...preIngestionEvent, properties: { $group_0: 'group_key' } }, + person, + processPerson + ) ) await eventsProcessor.kafkaProducer.flush() @@ -199,10 +203,12 @@ describe('EventsProcessor#createEvent()', () => { it('force_upgrade persons are recorded as such', async () => { const processPerson = false person.force_upgrade = true - eventsProcessor.createEvent( - { ...preIngestionEvent, properties: { $group_0: 'group_key' } }, - person, - processPerson + eventsProcessor.emitEvent( + eventsProcessor.createEvent( + { ...preIngestionEvent, properties: { $group_0: 'group_key' } }, + person, + processPerson + ) ) await eventsProcessor.kafkaProducer.flush() @@ -236,10 +242,12 @@ describe('EventsProcessor#createEvent()', () => { uuid: uuid, } const processPerson = true - eventsProcessor.createEvent( - { ...preIngestionEvent, distinctId: 'no-such-person' }, - nonExistingPerson, - processPerson + eventsProcessor.emitEvent( + eventsProcessor.createEvent( + { ...preIngestionEvent, distinctId: 'no-such-person' }, + nonExistingPerson, + processPerson + ) ) await eventsProcessor.kafkaProducer.flush() diff --git a/rust/common/kafka/src/kafka_producer.rs b/rust/common/kafka/src/kafka_producer.rs index 965406a1ce08c..666b7e72eeb55 100644 --- a/rust/common/kafka/src/kafka_producer.rs +++ b/rust/common/kafka/src/kafka_producer.rs @@ -90,15 +90,28 @@ pub async fn send_iter_to_kafka( topic: &str, iter: impl IntoIterator, ) -> Result<(), KafkaProduceError> +where + T: Serialize, +{ + send_keyed_iter_to_kafka(kafka_producer, topic, |_| None, iter).await +} + +pub async fn send_keyed_iter_to_kafka( + kafka_producer: &FutureProducer, + topic: &str, + key_extractor: impl Fn(&T) -> Option, + iter: impl IntoIterator, +) -> Result<(), KafkaProduceError> where T: Serialize, { let mut payloads = Vec::new(); for i in iter { + let key = key_extractor(&i); let payload = serde_json::to_string(&i) .map_err(|e| KafkaProduceError::SerializationError { error: e })?; - payloads.push(payload); + payloads.push((key, payload)); } if payloads.is_empty() { @@ -107,12 +120,12 @@ where let mut delivery_futures = Vec::new(); - for payload in payloads { + for (key, payload) in payloads { match kafka_producer.send_result(FutureRecord { topic, payload: Some(&payload), partition: None, - key: None::<&str>, + key: key.as_deref(), timestamp: None, headers: None, }) { diff --git a/rust/common/types/src/event.rs b/rust/common/types/src/event.rs index e678c87ae4273..5ce5efb62e8cf 100644 --- a/rust/common/types/src/event.rs +++ b/rust/common/types/src/event.rs @@ -39,6 +39,7 @@ pub enum PersonMode { pub struct ClickHouseEvent { pub uuid: Uuid, pub team_id: i32, + pub project_id: i32, pub event: String, pub distinct_id: String, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/rust/cymbal/src/app_context.rs b/rust/cymbal/src/app_context.rs index ef96c84fa7aa9..a57a0cb13201d 100644 --- a/rust/cymbal/src/app_context.rs +++ b/rust/cymbal/src/app_context.rs @@ -31,6 +31,7 @@ pub struct AppContext { pub pool: PgPool, pub catalog: Catalog, pub resolver: Resolver, + pub config: Config, } impl AppContext { @@ -103,6 +104,7 @@ impl AppContext { pool, catalog, resolver, + config: config.clone(), }) } } diff --git a/rust/cymbal/src/config.rs b/rust/cymbal/src/config.rs index 25a75a99e8e90..a6323556b7e91 100644 --- a/rust/cymbal/src/config.rs +++ b/rust/cymbal/src/config.rs @@ -12,6 +12,12 @@ pub struct Config { #[envconfig(nested = true)] pub kafka: KafkaConfig, + #[envconfig(default = "clickhouse_events_json")] + pub events_topic: String, + + #[envconfig(default = "clickhouse_error_tracking_issue_fingerprint")] + pub issue_overrides_topic: String, + #[envconfig(nested = true)] pub consumer: ConsumerConfig, diff --git a/rust/cymbal/src/issue_resolution.rs b/rust/cymbal/src/issue_resolution.rs index 6778ff100129d..b80c8d17d9b36 100644 --- a/rust/cymbal/src/issue_resolution.rs +++ b/rust/cymbal/src/issue_resolution.rs @@ -179,7 +179,6 @@ where conn.rollback().await?; } else { conn.commit().await?; - // TODO - emit new issue and override to kafka } Ok(fingerprinted.to_output(issue_override.issue_id)) diff --git a/rust/cymbal/src/lib.rs b/rust/cymbal/src/lib.rs index 09ea0a73f4928..6ab91b33c95de 100644 --- a/rust/cymbal/src/lib.rs +++ b/rust/cymbal/src/lib.rs @@ -22,13 +22,13 @@ pub mod types; pub async fn handle_event( context: Arc, mut event: ClickHouseEvent, -) -> Result, UnhandledError> { +) -> Result { let mut props = match get_props(&event) { Ok(r) => r, Err(e) => { warn!("Failed to get props: {}", e); add_error_to_event(&mut event, e)?; - return Ok(Some(event)); + return Ok(event); } }; @@ -37,7 +37,7 @@ pub async fn handle_event( if exceptions.is_empty() { props.add_error_message("No exceptions found on exception event"); event.properties = Some(serde_json::to_string(&props).unwrap()); - return Ok(Some(event)); + return Ok(event); } let mut results = Vec::new(); @@ -56,7 +56,7 @@ pub async fn handle_event( event.properties = Some(serde_json::to_string(&output).unwrap()); - Ok(Some(event)) + Ok(event) } fn get_props(event: &ClickHouseEvent) -> Result { diff --git a/rust/cymbal/src/main.rs b/rust/cymbal/src/main.rs index ef96e8e490ac4..62a479ae042be 100644 --- a/rust/cymbal/src/main.rs +++ b/rust/cymbal/src/main.rs @@ -1,7 +1,7 @@ use std::{future::ready, sync::Arc}; use axum::{routing::get, Router}; -use common_kafka::kafka_consumer::RecvErr; +use common_kafka::{kafka_consumer::RecvErr, kafka_producer::send_keyed_iter_to_kafka}; use common_metrics::{serve, setup_metrics_routes}; use common_types::ClickHouseEvent; use cymbal::{ @@ -78,11 +78,8 @@ async fn main() { }; metrics::counter!(EVENT_RECEIVED).increment(1); - let _processed_event = match handle_event(context.clone(), event).await { - Ok(r) => { - offset.store().unwrap(); - r - } + let event = match handle_event(context.clone(), event).await { + Ok(e) => e, Err(e) => { error!("Error handling event: {:?}", e); // If we get an unhandled error, it means we have some logical error in the code, or a @@ -91,7 +88,16 @@ async fn main() { } }; - // TODO - emit the event to the next Kafka topic + send_keyed_iter_to_kafka( + &context.kafka_producer, + &context.config.events_topic, + |ev| Some(ev.uuid.to_string()), + &[event], + ) + .await + .expect("Failed to send event to Kafka"); + + offset.store().unwrap(); metrics::counter!(STACK_PROCESSED).increment(1); whole_loop.label("finished", "true").fin(); diff --git a/rust/cymbal/tests/static/raw_ch_exception.json b/rust/cymbal/tests/static/raw_ch_exception.json index 2bb362f511dc6..382926b869708 100644 --- a/rust/cymbal/tests/static/raw_ch_exception.json +++ b/rust/cymbal/tests/static/raw_ch_exception.json @@ -4,6 +4,7 @@ "properties": "{\n \"$os\": \"Mac OS X\",\n \"$os_version\": \"10.15.7\",\n \"$browser\": \"Chrome\",\n \"$device_type\": \"Desktop\",\n \"$current_url\": \"https://us.posthog.com/project/2/feature_flags/50888\",\n \"$host\": \"us.posthog.com\",\n \"$pathname\": \"/project/2/feature_flags/50888\",\n \"$raw_user_agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36\",\n \"$browser_version\": 129,\n \"$browser_language\": \"en-US\",\n \"$screen_height\": 982,\n \"$screen_width\": 1512,\n \"$viewport_height\": 882,\n \"$viewport_width\": 1086,\n \"$lib\": \"web\",\n \"$lib_version\": \"1.166.0\",\n \"$insert_id\": \"xa7aywslwzwsilxn\",\n \"$time\": 1727960445.033,\n \"distinct_id\": \"UAnHBWWjwZPH8Ky85ZJ4BfYpMD45CvIVohyeJk97Wy9\",\n \"$device_id\": \"018fc4d4-4fc5-7e78-843d-f2d478a82f74\",\n \"$console_log_recording_enabled_server_side\": true,\n \"$session_recording_network_payload_capture\": {\n \"capturePerformance\": {\n \"network_timing\": true,\n \"web_vitals\": true,\n \"web_vitals_allowed_metrics\": null\n },\n \"recordBody\": true,\n \"recordHeaders\": true\n },\n \"$session_recording_canvas_recording\": {\n \"enabled\": false,\n \"fps\": null,\n \"quality\": null\n },\n \"$replay_sample_rate\": null,\n \"$replay_minimum_duration\": 2000,\n \"$initial_person_info\": {\n \"r\": \"$direct\",\n \"u\": \"https://us.posthog.com/project/84498/replay/01922131-3fc4-7c41-bc7a-fc4cd3cb5b12\"\n },\n \"$active_feature_flags\": [\n \"artificial-hog\",\n \"alerts\",\n \"hog-functions\",\n \"heatmaps-ui\",\n \"web-analytics-lcp-score\",\n \"error-tracking\"\n ],\n \"$feature/artificial-hog\": true,\n \"$feature/alerts\": true,\n \"$feature/hog-functions\": true,\n \"$feature/purchase-credits\": false,\n \"$feature/environments\": false,\n \"$feature/test-trend-juraj-1\": \"test\",\n \"$feature/onboarding-dashboard-templates\": \"control\",\n \"$feature_flag_payloads\": {\n \"changelog-notification\": \"{\\n \\\"notificationDate\\\": \\\"2024-08-26\\\", \\n \\\"markdown\\\": \\\"New this week: [React Native session replays](https://posthog.com/changelog/2024#react-native-session-replay-now-in-beta), [link Vitally to your data warehouse](https://posthog.com/changelog/2024#vitally-now-supported-as-a-warehouse-source), and [more](https://posthog.com/changelog)!\\\"\\n}\",\n \"string-payload\": \"true\",\n \"survey-test-target\": \"[1]\",\n \"product-ctas\": \"{\\\"title\\\": \\\"View pricing\\\", \\\"url\\\": \\\"pricing\\\"}\",\n \"available-plans\": \"{\\n \\\"planKeys\\\": {\\n \\\"marketing-website\\\": [\\\"123\\\", \\\"456\\\"],\\n \\\"app\\\": [\\\"789\\\", \\\"101112\\\"]\\n }\\n}\"\n },\n \"$user_id\": \"UAnHBWWjwZPH8Ky85ZJ4BfYpMD45CvIVohyeJk97Wy9\",\n \"is_demo_project\": false,\n \"$groups\": {\n \"project\": \"fc445b88-e2c4-488e-bb52-aa80cd7918c9\",\n \"organization\": \"4dc8564d-bd82-1065-2f40-97f7c50f67cf\",\n \"customer\": \"cus_IK2DWsWVn2ZM16\",\n \"instance\": \"https://us.posthog.com\"\n },\n \"realm\": \"cloud\",\n \"email_service_available\": true,\n \"slack_service_available\": true,\n \"commit_sha\": \"a71b50163b\",\n \"$autocapture_disabled_server_side\": false,\n \"$web_vitals_enabled_server_side\": true,\n \"$web_vitals_allowed_metrics\": null,\n \"$exception_capture_endpoint_suffix\": \"/e/\",\n \"$exception_capture_enabled_server_side\": true,\n \"has_billing_plan\": true,\n \"customer_deactivated\": false,\n \"current_total_amount_usd\": \"0.00\",\n \"custom_limits_usd.surveys\": 500,\n \"percentage_usage.product_analytics\": 0,\n \"current_amount_usd.product_analytics\": \"0.00\",\n \"unit_amount_usd.product_analytics\": null,\n \"usage_limit.product_analytics\": null,\n \"current_usage.product_analytics\": 13531521,\n \"projected_usage.product_analytics\": 225037109,\n \"free_allocation.product_analytics\": 0,\n \"percentage_usage.session_replay\": 0,\n \"current_amount_usd.session_replay\": \"0.00\",\n \"unit_amount_usd.session_replay\": null,\n \"usage_limit.session_replay\": null,\n \"current_usage.session_replay\": 70886,\n \"projected_usage.session_replay\": 1268192,\n \"free_allocation.session_replay\": 0,\n \"percentage_usage.feature_flags\": 0,\n \"current_amount_usd.feature_flags\": \"0.00\",\n \"unit_amount_usd.feature_flags\": null,\n \"usage_limit.feature_flags\": null,\n \"current_usage.feature_flags\": 16287759,\n \"projected_usage.feature_flags\": 461418955,\n \"free_allocation.feature_flags\": 0,\n \"percentage_usage.surveys\": 0,\n \"current_amount_usd.surveys\": \"0.00\",\n \"unit_amount_usd.surveys\": null,\n \"usage_limit.surveys\": null,\n \"current_usage.surveys\": 110,\n \"projected_usage.surveys\": 3047,\n \"free_allocation.surveys\": 0,\n \"percentage_usage.data_warehouse\": 0,\n \"current_amount_usd.data_warehouse\": \"0.00\",\n \"unit_amount_usd.data_warehouse\": null,\n \"usage_limit.data_warehouse\": null,\n \"current_usage.data_warehouse\": 109714956,\n \"projected_usage.data_warehouse\": 2121050491,\n \"free_allocation.data_warehouse\": 0,\n \"percentage_usage.integrations\": 0,\n \"current_amount_usd.integrations\": null,\n \"unit_amount_usd.integrations\": null,\n \"usage_limit.integrations\": 0,\n \"current_usage.integrations\": 0,\n \"projected_usage.integrations\": 0,\n \"free_allocation.integrations\": 0,\n \"percentage_usage.platform_and_support\": 0,\n \"current_amount_usd.platform_and_support\": null,\n \"unit_amount_usd.platform_and_support\": null,\n \"usage_limit.platform_and_support\": 0,\n \"current_usage.platform_and_support\": 0,\n \"projected_usage.platform_and_support\": 0,\n \"free_allocation.platform_and_support\": 0,\n \"billing_period_start\": {\n \"$L\": \"en\",\n \"$d\": {},\n \"$y\": 2024,\n \"$M\": 9,\n \"$D\": 2,\n \"$W\": 3,\n \"$H\": 17,\n \"$m\": 9,\n \"$s\": 56,\n \"$ms\": 0,\n \"$x\": {},\n \"$isDayjsObject\": true\n },\n \"billing_period_end\": {\n \"$L\": \"en\",\n \"$d\": {},\n \"$y\": 2024,\n \"$M\": 10,\n \"$D\": 2,\n \"$W\": 6,\n \"$H\": 16,\n \"$m\": 9,\n \"$s\": 56,\n \"$ms\": 0,\n \"$x\": {},\n \"$isDayjsObject\": true\n },\n \"$surveys_activated\": [\"0190fe51-92a0-0000-4ba3-f85f5f0ef78f\"],\n \"custom_limits_usd.data_warehouse\": 0,\n \"custom_limits_usd.feature_flags\": 0,\n \"custom_limits_usd.session_replay\": 0,\n \"custom_limits_usd.product_analytics\": 0,\n \"$referrer\": \"https://us.posthog.com/project/2/error_tracking\",\n \"$referring_domain\": \"us.posthog.com\",\n \"$exception_type\": \"Error\",\n \"$exception_message\": \"Unexpected usage\\n\\nError: Unexpected usage\\n at n.loadForeignModule (https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js:64:15003)\\n at https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js:64:25112\",\n \"$exception_stack_trace_raw\": \"[{\\\"filename\\\":\\\"https://app-static-prod.posthog.com/static/chunk-3LWGKVZR.js\\\",\\\"function\\\":\\\"r\\\",\\\"in_app\\\":true,\\\"lineno\\\":19,\\\"colno\\\":2044},{\\\"filename\\\":\\\"https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js\\\",\\\"function\\\":\\\"?\\\",\\\"in_app\\\":true,\\\"lineno\\\":3,\\\"colno\\\":12},{\\\"filename\\\":\\\"https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js\\\",\\\"function\\\":\\\"?\\\",\\\"in_app\\\":true,\\\"lineno\\\":64,\\\"colno\\\":25112},{\\\"filename\\\":\\\"https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js\\\",\\\"function\\\":\\\"n.loadForeignModule\\\",\\\"in_app\\\":true,\\\"lineno\\\":64,\\\"colno\\\":15003}]\",\n \"$exception_level\": \"error\",\n \"$exception_source\": \"https://app-static-prod.posthog.com/static/chunk-3LWGKVZR.js\",\n \"$exception_lineno\": 19,\n \"$exception_colno\": 2067,\n \"$exception_personURL\": \"https://us.posthog.com/project/sTMFPsFhdP1Ssg/person/UAnHBWWjwZPH8Ky85ZJ4BfYpMD45CvIVohyeJk97Wy9\",\n \"token\": \"sYTIHGHJKJHGFHJG\",\n \"$session_id\": \"01925255-fc87-771a-8aef-607444f71b4c\",\n \"$window_id\": \"01925273-b871-75c4-bd8f-54a978ead4f9\",\n \"$lib_custom_api_host\": \"https://internal-t.posthog.com\",\n \"$is_identified\": true,\n \"$lib_rate_limit_remaining_tokens\": 98.02,\n \"$set_once\": {\n \"$initial_os\": \"Mac OS X\",\n \"$initial_os_version\": \"10.15.7\",\n \"$initial_browser\": \"Chrome\",\n \"$initial_device_type\": \"Desktop\",\n \"$initial_current_url\": \"https://us.posthog.com/project/84498/replay/01922131-3fc4-7c41-bc7a-fc4cd3cb5b12\",\n \"$initial_pathname\": \"/project/84498/replay/01922131-3fc4-7c41-bc7a-fc4cd3cb5b12\",\n \"$initial_browser_version\": 129,\n \"$initial_referrer\": \"$direct\",\n \"$initial_referring_domain\": \"$direct\",\n \"$initial_geoip_city_name\": \"London\",\n \"$initial_geoip_city_confidence\": null,\n \"$initial_geoip_subdivision_2_name\": null,\n \"$initial_geoip_subdivision_2_code\": null,\n \"$initial_geoip_subdivision_2_confidence\": null,\n \"$initial_geoip_subdivision_1_name\": \"England\",\n \"$initial_geoip_subdivision_1_code\": \"ENG\",\n \"$initial_geoip_subdivision_1_confidence\": null,\n \"$initial_geoip_country_name\": \"United Kingdom\",\n \"$initial_geoip_country_code\": \"GB\",\n \"$initial_geoip_country_confidence\": null,\n \"$initial_geoip_continent_name\": \"Europe\",\n \"$initial_geoip_continent_code\": \"EU\",\n \"$initial_geoip_postal_code\": \"EC4R\",\n \"$initial_geoip_postal_code_confidence\": null,\n \"$initial_geoip_latitude\": 51.5088,\n \"$initial_geoip_longitude\": -0.093,\n \"$initial_geoip_accuracy_radius\": 20,\n \"$initial_geoip_time_zone\": \"Europe/London\",\n \"$initial_host\": \"us.posthog.com\"\n },\n \"$ip\": \"62.23.207.10\",\n \"$set\": {\n \"$os\": \"Mac OS X\",\n \"$os_version\": \"10.15.7\",\n \"$browser\": \"Chrome\",\n \"$device_type\": \"Desktop\",\n \"$current_url\": \"https://us.posthog.com/project/2/feature_flags/50888\",\n \"$pathname\": \"/project/2/feature_flags/50888\",\n \"$browser_version\": 129,\n \"$referrer\": \"https://us.posthog.com/project/2/error_tracking\",\n \"$referring_domain\": \"us.posthog.com\",\n \"$geoip_city_name\": \"London\",\n \"$geoip_city_confidence\": null,\n \"$geoip_subdivision_2_name\": null,\n \"$geoip_subdivision_2_code\": null,\n \"$geoip_subdivision_2_confidence\": null,\n \"$geoip_subdivision_1_name\": \"England\",\n \"$geoip_subdivision_1_code\": \"ENG\",\n \"$geoip_subdivision_1_confidence\": null,\n \"$geoip_country_name\": \"United Kingdom\",\n \"$geoip_country_code\": \"GB\",\n \"$geoip_country_confidence\": null,\n \"$geoip_continent_name\": \"Europe\",\n \"$geoip_continent_code\": \"EU\",\n \"$geoip_postal_code\": \"EC4R\",\n \"$geoip_postal_code_confidence\": null,\n \"$geoip_latitude\": 51.5088,\n \"$geoip_longitude\": -0.093,\n \"$geoip_accuracy_radius\": 20,\n \"$geoip_time_zone\": \"Europe/London\"\n },\n \"$sent_at\": \"2024-10-03T13:00:45.158000+00:00\",\n \"$geoip_city_name\": \"London\",\n \"$geoip_city_confidence\": null,\n \"$geoip_country_name\": \"United Kingdom\",\n \"$geoip_country_code\": \"GB\",\n \"$geoip_country_confidence\": null,\n \"$geoip_continent_name\": \"Europe\",\n \"$geoip_continent_code\": \"EU\",\n \"$geoip_postal_code\": \"EC4R\",\n \"$geoip_postal_code_confidence\": null,\n \"$geoip_latitude\": 51.5088,\n \"$geoip_longitude\": -0.093,\n \"$geoip_accuracy_radius\": 20,\n \"$geoip_time_zone\": \"Europe/London\",\n \"$geoip_subdivision_1_code\": \"ENG\",\n \"$geoip_subdivision_1_name\": \"England\",\n \"$geoip_subdivision_1_confidence\": null,\n \"$lib_version__major\": 1,\n \"$lib_version__minor\": 166,\n \"$lib_version__patch\": 0,\n \"$group_2\": \"fc445b88-e2c4-488e-bb52-aa80cd7918c9\",\n \"$group_0\": \"4dc8564d-bd82-1065-2f40-97f7c50f67cf\",\n \"$group_3\": \"cus_IK2DWsWVn2ZM16\",\n \"$group_1\": \"https://us.posthog.com\",\n \"$exception_fingerprint\": [\n \"Error\",\n \"Unexpected usage\\n\\nError: Unexpected usage\\n at n.loadForeignModule (https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js:64:15003)\\n at https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js:64:25112\",\n \"r\"\n ]\n }", "timestamp": "2024-10-03T06:00:45.069000-07:00", "team_id": 2, + "project_id": 2, "distinct_id": "UAnHBWWjwZPH8Ky85ZJ4BfYpMD45CvIVohyeJk97Wy9", "elements_chain": "", "created_at": "2024-10-03T06:01:25.266000-07:00", diff --git a/rust/cymbal/tests/static/raw_ch_exception_list.json b/rust/cymbal/tests/static/raw_ch_exception_list.json index 1c8eebbdc97e8..78b961dfdbcf7 100644 --- a/rust/cymbal/tests/static/raw_ch_exception_list.json +++ b/rust/cymbal/tests/static/raw_ch_exception_list.json @@ -4,6 +4,7 @@ "properties": "{\"$os\":\"Mac OS X\",\"$os_version\":\"10.15.7\",\"$browser\":\"Chrome\",\"$device_type\":\"Desktop\",\"$current_url\":\"https://eu.posthog.com/project/12557/feature_flags/31624\",\"$host\":\"eu.posthog.com\",\"$pathname\":\"/project/12557/feature_flags/31624\",\"$raw_user_agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36\",\"$browser_version\":129,\"$browser_language\":\"en-GB\",\"$screen_height\":1080,\"$screen_width\":1920,\"$viewport_height\":934,\"$viewport_width\":1920,\"$lib\":\"web\",\"$lib_version\":\"1.170.1\",\"$insert_id\":\"xjfjg606eo2x7n4x\",\"$time\":1729088278.943,\"distinct_id\":\"pQC9X9Fe7BPzJXVxpY0fx37UwFOCd1vXHzh8rjUPv1G\",\"$device_id\":\"018ccedb-d598-79bb-94e0-4751a3b956f4\",\"$console_log_recording_enabled_server_side\":true,\"$autocapture_disabled_server_side\":false,\"$web_vitals_enabled_server_side\":true,\"$exception_capture_enabled_server_side\":true,\"$exception_capture_endpoint\":\"/e/\",\"realm\":\"cloud\",\"email_service_available\":true,\"slack_service_available\":true,\"commit_sha\":\"bafa32953e\",\"$user_id\":\"pQC9X9Fe7BPzJXVxpY0fx37UwFOCd1vXHzh8rjUPv1G\",\"is_demo_project\":false,\"$groups\":{\"project\":\"018c1057-288d-0000-93bb-3bd44c845f22\",\"organization\":\"018afaa6-8b2e-0000-2311-d58d2df832ad\",\"customer\":\"cus_P5B9QmoUKLAUlx\",\"instance\":\"https://eu.posthog.com\"},\"has_billing_plan\":true,\"$referrer\":\"$direct\",\"$referring_domain\":\"$direct\",\"$session_recording_start_reason\":\"session_id_changed\",\"$exception_list\":[{\"type\":\"UnhandledRejection\",\"value\":\"Unexpected usage\",\"stacktrace\":{\"type\": \"raw\", \"frames\":[{\"filename\":\"https://app-static.eu.posthog.com/static/chunk-PGUQKT6S.js\",\"function\":\"?\",\"in_app\":true,\"lineno\":64,\"colno\":25112},{\"filename\":\"https://app-static.eu.posthog.com/static/chunk-PGUQKT6S.js\",\"function\":\"n.loadForeignModule\",\"in_app\":true,\"lineno\":64,\"colno\":15003}]},\"mechanism\":{\"handled\":false,\"synthetic\":false}}],\"$exception_level\":\"error\",\"$exception_personURL\":\"https://us.posthog.com/project/sTMFPsFhdP1Ssg/person/pQC9X9Fe7BPzJXVxpY0fx37UwFOCd1vXHzh8rjUPv1G\",\"token\":\"sTMFPsFhdP1Ssg\",\"$session_id\":\"019295b0-db2b-7e02-8010-0a1c4db680df\",\"$window_id\":\"019295b0-db2b-7e02-8010-0a1dee88e5f5\",\"$lib_custom_api_host\":\"https://internal-t.posthog.com\",\"$is_identified\":true,\"$lib_rate_limit_remaining_tokens\":97.28999999999999,\"$sent_at\":\"2024-10-16T14:17:59.543000+00:00\",\"$geoip_city_name\":\"Lisbon\",\"$geoip_city_confidence\":null,\"$geoip_country_name\":\"Portugal\",\"$geoip_country_code\":\"PT\",\"$geoip_country_confidence\":null,\"$geoip_continent_name\":\"Europe\",\"$geoip_continent_code\":\"EU\",\"$geoip_postal_code\":\"1269-001\",\"$geoip_postal_code_confidence\":null,\"$geoip_latitude\":38.731,\"$geoip_longitude\":-9.1373,\"$geoip_accuracy_radius\":100,\"$geoip_time_zone\":\"Europe/Lisbon\",\"$geoip_subdivision_1_code\":\"11\",\"$geoip_subdivision_1_name\":\"Lisbon\",\"$geoip_subdivision_1_confidence\":null,\"$lib_version__major\":1,\"$lib_version__minor\":170,\"$lib_version__patch\":1,\"$group_2\":\"018c1057-288d-0000-93bb-3bd44c845f22\",\"$group_0\":\"018afaa6-8b2e-0000-2311-d58d2df832ad\",\"$group_3\":\"cus_P5B9QmoUKLAUlx\",\"$group_1\":\"https://eu.posthog.com\"}", "timestamp": "2024-10-16T07:17:59.088000-07:00", "team_id": 2, + "project_id": 2, "distinct_id": "pQC9X9Fe7BPzJXVxpY0fx37UwFOCd1vXHzh8rjUPv1G", "elements_chain": "", "created_at": "2024-10-16T07:18:00.100000-07:00",