From 6183f88b67b4d6203ce53ad8a1062c9e3dc01b8a Mon Sep 17 00:00:00 2001 From: Ben White Date: Tue, 23 Apr 2024 09:24:26 +0200 Subject: [PATCH] feat: Heatmaps ingestion (#21629) --- plugin-server/src/config/config.ts | 2 + plugin-server/src/config/kafka-topics.ts | 2 + plugin-server/src/types.ts | 24 + .../event-pipeline/extractHeatmapDataStep.ts | 126 + .../worker/ingestion/event-pipeline/runner.ts | 13 +- .../extractHeatmapDataStep.test.ts.snap | 5208 +++++++++++++++++ .../__snapshots__/runner.test.ts.snap | 15 + .../extractHeatmapDataStep.test.ts | 226 + .../ingestion/event-pipeline/runner.test.ts | 4 +- 9 files changed, 5618 insertions(+), 2 deletions(-) create mode 100644 plugin-server/src/worker/ingestion/event-pipeline/extractHeatmapDataStep.ts create mode 100644 plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/extractHeatmapDataStep.test.ts.snap create mode 100644 plugin-server/tests/worker/ingestion/event-pipeline/extractHeatmapDataStep.test.ts diff --git a/plugin-server/src/config/config.ts b/plugin-server/src/config/config.ts index ea51dc1da394d..1d5c1ff360162 100644 --- a/plugin-server/src/config/config.ts +++ b/plugin-server/src/config/config.ts @@ -2,6 +2,7 @@ import { LogLevel, PluginLogLevel, PluginsServerConfig, stringToPluginServerMode import { isDevEnv, isTestEnv, stringToBoolean } from '../utils/env-utils' import { KAFKAJS_LOG_LEVEL_MAPPING } from './constants' import { + KAFKA_CLICKHOUSE_HEATMAP_EVENTS, KAFKA_EVENTS_JSON, KAFKA_EVENTS_PLUGIN_INGESTION, KAFKA_EVENTS_PLUGIN_INGESTION_OVERFLOW, @@ -104,6 +105,7 @@ export function getDefaultConfig(): PluginsServerConfig { KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY: 1, CLICKHOUSE_DISABLE_EXTERNAL_SCHEMAS_TEAMS: '', CLICKHOUSE_JSON_EVENTS_KAFKA_TOPIC: KAFKA_EVENTS_JSON, + CLICKHOUSE_HEATMAPS_KAFKA_TOPIC: KAFKA_CLICKHOUSE_HEATMAP_EVENTS, CONVERSION_BUFFER_ENABLED: false, CONVERSION_BUFFER_ENABLED_TEAMS: '', CONVERSION_BUFFER_TOPIC_ENABLED_TEAMS: '', diff --git a/plugin-server/src/config/kafka-topics.ts b/plugin-server/src/config/kafka-topics.ts index 71f9bd8ee79da..d93fa8f897afe 100644 --- a/plugin-server/src/config/kafka-topics.ts +++ b/plugin-server/src/config/kafka-topics.ts @@ -36,6 +36,8 @@ export const KAFKA_CLICKHOUSE_SESSION_RECORDING_EVENTS = `${prefix}clickhouse_se export const KAFKA_CLICKHOUSE_SESSION_REPLAY_EVENTS = `${prefix}clickhouse_session_replay_events${suffix}` // write performance events to ClickHouse export const KAFKA_PERFORMANCE_EVENTS = `${prefix}clickhouse_performance_events${suffix}` +// write heatmap events to ClickHouse +export const KAFKA_CLICKHOUSE_HEATMAP_EVENTS = `${prefix}clickhouse_heatmap_events${suffix}` // log entries for ingestion into clickhouse export const KAFKA_LOG_ENTRIES = `${prefix}log_entries${suffix}` diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index 4d09e06de4ba1..31c25a13025f1 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -116,6 +116,7 @@ export interface PluginsServerConfig { CLICKHOUSE_DISABLE_EXTERNAL_SCHEMAS: boolean // whether to disallow external schemas like protobuf for clickhouse kafka engine CLICKHOUSE_DISABLE_EXTERNAL_SCHEMAS_TEAMS: string // (advanced) a comma separated list of teams to disable clickhouse external schemas for CLICKHOUSE_JSON_EVENTS_KAFKA_TOPIC: string // (advanced) topic to send events to for clickhouse ingestion + CLICKHOUSE_HEATMAPS_KAFKA_TOPIC: string // (advanced) topic to send heatmap data to for clickhouse ingestion REDIS_URL: string POSTHOG_REDIS_PASSWORD: string POSTHOG_REDIS_HOST: string @@ -1099,3 +1100,26 @@ export type RRWebEvent = Record & { export interface ValueMatcher { (value: T): boolean } + +export type RawClickhouseHeatmapEvent = { + /** + * session id lets us offer example recordings on high traffic parts of the page, + * and could let us offer more advanced filtering of heatmap data + * we will break the relationship between particular sessions and clicks in aggregating this data + * it should always be treated as an exemplar and not as concrete values + */ + session_id: string + distinct_id: string + viewport_width: number + viewport_height: number + pointer_target_fixed: boolean + current_url: string + // x is the x with resolution applied, the resolution converts high fidelity mouse positions into an NxN grid + x: number + // y is the y with resolution applied, the resolution converts high fidelity mouse positions into an NxN grid + y: number + scale_factor: 16 // in the future we may support other values + timestamp: string + type: string + team_id: number +} diff --git a/plugin-server/src/worker/ingestion/event-pipeline/extractHeatmapDataStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/extractHeatmapDataStep.ts new file mode 100644 index 0000000000000..3b16437da81f2 --- /dev/null +++ b/plugin-server/src/worker/ingestion/event-pipeline/extractHeatmapDataStep.ts @@ -0,0 +1,126 @@ +import { URL } from 'url' + +import { PreIngestionEvent, RawClickhouseHeatmapEvent, TimestampFormat } from '../../../types' +import { castTimestampOrNow } from '../../../utils/utils' +import { captureIngestionWarning } from '../utils' +import { EventPipelineRunner } from './runner' + +// This represents the scale factor for the heatmap data. Essentially how much we are reducing the resolution by. +const SCALE_FACTOR = 16 + +type HeatmapDataItem = { + x: number + y: number + target_fixed: boolean + type: string +} + +type HeatmapData = Record + +export function extractHeatmapDataStep( + runner: EventPipelineRunner, + event: PreIngestionEvent +): Promise<[PreIngestionEvent, Promise[]]> { + const { eventUuid, teamId } = event + + let acks: Promise[] = [] + + try { + const heatmapEvents = extractScrollDepthHeatmapData(event) ?? [] + + // eslint-disable-next-line @typescript-eslint/no-floating-promises + acks = heatmapEvents.map((rawEvent) => { + return runner.hub.kafkaProducer.produce({ + topic: runner.hub.CLICKHOUSE_HEATMAPS_KAFKA_TOPIC, + key: eventUuid, + value: Buffer.from(JSON.stringify(rawEvent)), + waitForAck: true, + }) + }) + } catch (e) { + acks.push( + captureIngestionWarning(runner.hub.kafkaProducer, teamId, 'invalid_heatmap_data', { + eventUuid, + }) + ) + } + + // We don't want to ingest this data to the events table + delete event.properties['$heatmap_data'] + + return Promise.resolve([event, acks]) +} + +function replacePathInUrl(url: string, newPath: string): string { + const parsedUrl = new URL(url) + parsedUrl.pathname = newPath + return parsedUrl.toString() +} + +function extractScrollDepthHeatmapData(event: PreIngestionEvent): RawClickhouseHeatmapEvent[] { + const { teamId, timestamp, properties } = event + const { + $viewport_height, + $viewport_width, + $session_id, + distinct_id, + $prev_pageview_pathname, + $prev_pageview_max_scroll, + $current_url, + $heatmap_data, + } = properties || {} + + let heatmapData = $heatmap_data as HeatmapData | null + + if ($prev_pageview_pathname && $current_url) { + // We are going to add the scroll depth info derived from the previous pageview to the current pageview's heatmap data + if (!heatmapData) { + heatmapData = {} + } + + const previousUrl = replacePathInUrl($current_url, $prev_pageview_pathname) + heatmapData[previousUrl] = heatmapData[previousUrl] || [] + heatmapData[previousUrl].push({ + x: 0, + y: $prev_pageview_max_scroll, + target_fixed: false, + type: 'scrolldepth', + }) + } + + let heatmapEvents: RawClickhouseHeatmapEvent[] = [] + + if (!heatmapData) { + return [] + } + + Object.entries(heatmapData).forEach(([url, items]) => { + if (Array.isArray(items)) { + heatmapEvents = heatmapEvents.concat( + (items as any[]).map( + (hme: { + x: number + y: number + target_fixed: boolean + type: string + }): RawClickhouseHeatmapEvent => ({ + type: hme.type, + x: Math.round(hme.x / SCALE_FACTOR), + y: Math.round(hme.y / SCALE_FACTOR), + pointer_target_fixed: hme.target_fixed, + viewport_height: Math.round($viewport_height / SCALE_FACTOR), + viewport_width: Math.round($viewport_width / SCALE_FACTOR), + current_url: url, + session_id: $session_id, + scale_factor: SCALE_FACTOR, + timestamp: castTimestampOrNow(timestamp ?? null, TimestampFormat.ClickHouse), + team_id: teamId, + distinct_id: distinct_id, + }) + ) + ) + } + }) + + return heatmapEvents +} diff --git a/plugin-server/src/worker/ingestion/event-pipeline/runner.ts b/plugin-server/src/worker/ingestion/event-pipeline/runner.ts index 9ad11391f0c2f..52e762949a924 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/runner.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/runner.ts @@ -10,6 +10,7 @@ import { normalizeProcessPerson } from '../../../utils/event' import { status } from '../../../utils/status' import { captureIngestionWarning, generateEventDeadLetterQueueMessage } from '../utils' import { createEventStep } from './createEventStep' +import { extractHeatmapDataStep } from './extractHeatmapDataStep' import { eventProcessedAndIngestedCounter, pipelineLastStepCounter, @@ -216,9 +217,19 @@ export class EventPipelineRunner { event.team_id ) + const [preparedEventWithoutHeatmaps, heatmapKafkaAcks] = await this.runStep( + extractHeatmapDataStep, + [this, preparedEvent], + event.team_id + ) + + if (heatmapKafkaAcks.length > 0) { + kafkaAcks.push(...heatmapKafkaAcks) + } + const [rawClickhouseEvent, eventAck] = await this.runStep( createEventStep, - [this, preparedEvent, person, processPerson], + [this, preparedEventWithoutHeatmaps, person, processPerson], event.team_id ) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/extractHeatmapDataStep.test.ts.snap b/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/extractHeatmapDataStep.test.ts.snap new file mode 100644 index 0000000000000..12238bd7ab350 --- /dev/null +++ b/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/extractHeatmapDataStep.test.ts.snap @@ -0,0 +1,5208 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`extractHeatmapDataStep() parses and ingests correct $heatmap_data 2`] = ` +Array [ + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 109, + 111, + 117, + 115, + 101, + 109, + 111, + 118, + 101, + 34, + 44, + 34, + 120, + 34, + 58, + 54, + 52, + 44, + 34, + 121, + 34, + 58, + 50, + 51, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 114, + 97, + 103, + 101, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 109, + 111, + 117, + 115, + 101, + 109, + 111, + 118, + 101, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 109, + 111, + 117, + 115, + 101, + 109, + 111, + 118, + 101, + 34, + 44, + 34, + 120, + 34, + 58, + 54, + 54, + 44, + 34, + 121, + 34, + 58, + 52, + 50, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 55, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 55, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 114, + 97, + 103, + 101, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 55, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 48, + 44, + 34, + 121, + 34, + 58, + 50, + 55, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 52, + 53, + 44, + 34, + 121, + 34, + 58, + 49, + 55, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], + Array [ + Object { + "key": "018eebf3-cb48-750b-bfad-36409ea6f2b2", + "topic": undefined, + "value": Object { + "data": Array [ + 123, + 34, + 116, + 121, + 112, + 101, + 34, + 58, + 34, + 99, + 108, + 105, + 99, + 107, + 34, + 44, + 34, + 120, + 34, + 58, + 55, + 44, + 34, + 121, + 34, + 58, + 57, + 44, + 34, + 112, + 111, + 105, + 110, + 116, + 101, + 114, + 95, + 116, + 97, + 114, + 103, + 101, + 116, + 95, + 102, + 105, + 120, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 104, + 101, + 105, + 103, + 104, + 116, + 34, + 58, + 56, + 51, + 44, + 34, + 118, + 105, + 101, + 119, + 112, + 111, + 114, + 116, + 95, + 119, + 105, + 100, + 116, + 104, + 34, + 58, + 54, + 55, + 44, + 34, + 99, + 117, + 114, + 114, + 101, + 110, + 116, + 95, + 117, + 114, + 108, + 34, + 58, + 34, + 104, + 116, + 116, + 112, + 58, + 47, + 47, + 108, + 111, + 99, + 97, + 108, + 104, + 111, + 115, + 116, + 58, + 51, + 48, + 48, + 48, + 47, + 34, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 99, + 100, + 45, + 55, + 48, + 100, + 97, + 45, + 56, + 57, + 53, + 102, + 45, + 98, + 54, + 99, + 102, + 51, + 53, + 50, + 98, + 100, + 54, + 56, + 56, + 34, + 44, + 34, + 115, + 99, + 97, + 108, + 101, + 95, + 102, + 97, + 99, + 116, + 111, + 114, + 34, + 58, + 49, + 54, + 44, + 34, + 116, + 105, + 109, + 101, + 115, + 116, + 97, + 109, + 112, + 34, + 58, + 34, + 50, + 48, + 50, + 52, + 45, + 48, + 52, + 45, + 49, + 55, + 32, + 49, + 50, + 58, + 48, + 54, + 58, + 52, + 54, + 46, + 56, + 54, + 49, + 34, + 44, + 34, + 116, + 101, + 97, + 109, + 95, + 105, + 100, + 34, + 58, + 49, + 44, + 34, + 100, + 105, + 115, + 116, + 105, + 110, + 99, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 56, + 101, + 101, + 98, + 102, + 51, + 45, + 55, + 57, + 98, + 49, + 45, + 55, + 48, + 56, + 50, + 45, + 97, + 55, + 99, + 54, + 45, + 101, + 101, + 98, + 53, + 54, + 97, + 51, + 54, + 48, + 48, + 50, + 102, + 34, + 125, + ], + "type": "Buffer", + }, + "waitForAck": true, + }, + ], +] +`; diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/runner.test.ts.snap b/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/runner.test.ts.snap index 9cd0d244500ae..fd5ccf04db02d 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/runner.test.ts.snap +++ b/plugin-server/tests/worker/ingestion/event-pipeline/__snapshots__/runner.test.ts.snap @@ -80,6 +80,21 @@ Array [ true, ], ], + Array [ + "extractHeatmapDataStep", + Array [ + Object { + "distinctId": "my_id", + "elementsList": Array [], + "event": "$pageview", + "eventUuid": "uuid1", + "ip": "127.0.0.1", + "properties": Object {}, + "teamId": 2, + "timestamp": "2020-02-23T02:15:00.000Z", + }, + ], + ], Array [ "createEventStep", Array [ diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/extractHeatmapDataStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/extractHeatmapDataStep.test.ts new file mode 100644 index 0000000000000..ef654deb0df36 --- /dev/null +++ b/plugin-server/tests/worker/ingestion/event-pipeline/extractHeatmapDataStep.test.ts @@ -0,0 +1,226 @@ +import { ISOTimestamp, PreIngestionEvent } from '../../../../src/types' +import { cloneObject } from '../../../../src/utils/utils' +import { extractHeatmapDataStep } from '../../../../src/worker/ingestion/event-pipeline/extractHeatmapDataStep' + +jest.mock('../../../../src/worker/plugins/run') + +const preIngestionEvent: PreIngestionEvent = { + eventUuid: '018eebf3-cb48-750b-bfad-36409ea6f2b2', + event: 'Clicked button', + distinctId: '018eebf3-79b1-7082-a7c6-eeb56a36002f', + properties: { + $current_url: 'http://localhost:3000/', + $host: 'localhost:3000', + $pathname: '/', + $viewport_height: 1328, + $viewport_width: 1071, + distinct_id: '018eebf3-79b1-7082-a7c6-eeb56a36002f', + $device_id: '018eebf3-79b1-7082-a7c6-eeb56a36002f', + $session_id: '018eebf3-79cd-70da-895f-b6cf352bd688', + $window_id: '018eebf3-79cd-70da-895f-b6d09add936a', + $heatmap_data: { + 'http://localhost:3000/': [ + { + x: 1020, + y: 363, + target_fixed: false, + type: 'mousemove', + }, + { + x: 634, + y: 460, + target_fixed: false, + type: 'click', + }, + { + x: 634, + y: 460, + target_fixed: false, + type: 'click', + }, + { + x: 634, + y: 460, + target_fixed: false, + type: 'rageclick', + }, + { + x: 634, + y: 460, + target_fixed: false, + type: 'click', + }, + { + x: 634, + y: 460, + target_fixed: false, + type: 'click', + }, + { + x: 634, + y: 460, + target_fixed: false, + type: 'click', + }, + { + x: 634, + y: 460, + target_fixed: false, + type: 'click', + }, + { + x: 634, + y: 460, + target_fixed: false, + type: 'mousemove', + }, + { + x: 1052, + y: 665, + target_fixed: false, + type: 'mousemove', + }, + { + x: 632, + y: 436, + target_fixed: false, + type: 'click', + }, + { + x: 632, + y: 436, + target_fixed: false, + type: 'click', + }, + { + x: 632, + y: 436, + target_fixed: false, + type: 'rageclick', + }, + { + x: 632, + y: 436, + target_fixed: false, + type: 'click', + }, + { + x: 713, + y: 264, + target_fixed: false, + type: 'click', + }, + { + x: 119, + y: 143, + target_fixed: false, + type: 'click', + }, + ], + }, + }, + timestamp: '2024-04-17T12:06:46.861Z' as ISOTimestamp, + teamId: 1, +} + +describe('extractHeatmapDataStep()', () => { + let runner: any + let event: PreIngestionEvent + + beforeEach(() => { + event = cloneObject(preIngestionEvent) + runner = { + hub: { + kafkaProducer: { + produce: jest.fn((e) => Promise.resolve(e)), + }, + }, + nextStep: (...args: any[]) => args, + } + }) + + it('parses and ingests correct $heatmap_data', async () => { + const response = await extractHeatmapDataStep(runner, event) + expect(response[0]).toEqual(event) + expect(response[0].properties.$heatmap_data).toBeUndefined() + expect(response[1]).toHaveLength(16) + expect(runner.hub.kafkaProducer.produce).toBeCalledTimes(16) + + const firstProduceCall = runner.hub.kafkaProducer.produce.mock.calls[0][0] + + const parsed = JSON.parse(firstProduceCall.value.toString()) + + expect(parsed).toMatchInlineSnapshot(` + Object { + "current_url": "http://localhost:3000/", + "distinct_id": "018eebf3-79b1-7082-a7c6-eeb56a36002f", + "pointer_target_fixed": false, + "scale_factor": 16, + "session_id": "018eebf3-79cd-70da-895f-b6cf352bd688", + "team_id": 1, + "timestamp": "2024-04-17 12:06:46.861", + "type": "mousemove", + "viewport_height": 83, + "viewport_width": 67, + "x": 64, + "y": 23, + } + `) + + // The rest we can just compare the buffers + expect(runner.hub.kafkaProducer.produce.mock.calls).toMatchSnapshot() + }) + + it('ignores events without $heatmap_data', async () => { + event.properties.$heatmap_data = null + const response = await extractHeatmapDataStep(runner, event) + expect(response).toEqual([event, []]) + expect(response[0].properties.$heatmap_data).toBeUndefined() + }) + + it('ignores events with bad $heatmap_data', async () => { + event.properties.$heatmap_data = 'wat' + const response = await extractHeatmapDataStep(runner, event) + expect(response).toEqual([event, []]) + expect(response[0].properties.$heatmap_data).toBeUndefined() + }) + + it('additionally parses ', async () => { + event.properties = { + ...event.properties, + $prev_pageview_pathname: '/test', + $prev_pageview_max_scroll: 225, + $prev_pageview_last_content: 1445, + $prev_pageview_max_content: 1553, + } + + const response = await extractHeatmapDataStep(runner, event) + // We do delete heatmap data + expect(response[0].properties.$heatmap_data).toBeUndefined() + // We don't delete scroll properties + expect(response[0].properties.$prev_pageview_max_scroll).toEqual(225) + + expect(response[1]).toHaveLength(17) + + const allParsedMessages = runner.hub.kafkaProducer.produce.mock.calls.map((call) => + JSON.parse(call[0].value.toString()) + ) + + expect(allParsedMessages.find((x) => x.type === 'scrolldepth')).toMatchInlineSnapshot(` + Object { + "current_url": "http://localhost:3000/test", + "distinct_id": "018eebf3-79b1-7082-a7c6-eeb56a36002f", + "pointer_target_fixed": false, + "scale_factor": 16, + "session_id": "018eebf3-79cd-70da-895f-b6cf352bd688", + "team_id": 1, + "timestamp": "2024-04-17 12:06:46.861", + "type": "scrolldepth", + "viewport_height": 83, + "viewport_width": 67, + "x": 0, + "y": 14, + } + `) + }) +}) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts index 818d96f2656dc..0cf0fcdb08cd5 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts @@ -120,6 +120,7 @@ describe('EventPipelineRunner', () => { 'normalizeEventStep', 'processPersonsStep', 'prepareEventStep', + 'extractHeatmapDataStep', 'createEventStep', ]) expect(runner.stepsWithArgs).toMatchSnapshot() @@ -147,6 +148,7 @@ describe('EventPipelineRunner', () => { 'normalizeEventStep', 'processPersonsStep', 'prepareEventStep', + 'extractHeatmapDataStep', 'createEventStep', ]) }) @@ -169,7 +171,7 @@ describe('EventPipelineRunner', () => { const result = await runner.runEventPipeline(pipelineEvent) expect(result.error).toBeUndefined() - expect(pipelineStepMsSummarySpy).toHaveBeenCalledTimes(6) + expect(pipelineStepMsSummarySpy).toHaveBeenCalledTimes(7) expect(pipelineLastStepCounterSpy).toHaveBeenCalledTimes(1) expect(eventProcessedAndIngestedCounterSpy).toHaveBeenCalledTimes(1) expect(pipelineStepMsSummarySpy).toHaveBeenCalledWith('createEventStep')