From e452ef0e8f7be5d0c54044af7fbd88c33f53d996 Mon Sep 17 00:00:00 2001 From: Tiina Turban Date: Fri, 6 Oct 2023 13:50:08 +0200 Subject: [PATCH] feat: Ingestion to accept elements_chain string (#17809) Co-authored-by: Brett Hoerner --- plugin-server/src/types.ts | 12 +++- .../src/worker/ingestion/process-event.ts | 55 +++++++++++++------ .../event-pipeline/prepareEventStep.test.ts | 43 ++++++++++++++- 3 files changed, 88 insertions(+), 22 deletions(-) diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index 525e4a08853cf..83cf936a3b713 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -658,8 +658,16 @@ interface BaseIngestionEvent { elementsList: Element[] } -/** Ingestion event before saving, currently just an alias of BaseIngestionEvent. */ -export type PreIngestionEvent = BaseIngestionEvent +/** Ingestion event before saving, BaseIngestionEvent without elementsList */ +export interface PreIngestionEvent { + eventUuid: string + event: string + ip: string | null + teamId: TeamId + distinctId: string + properties: Properties + timestamp: ISOTimestamp +} /** Ingestion event after saving, currently just an alias of BaseIngestionEvent */ export interface PostIngestionEvent extends BaseIngestionEvent { diff --git a/plugin-server/src/worker/ingestion/process-event.ts b/plugin-server/src/worker/ingestion/process-event.ts index 92480ce849994..4b0397c66d792 100644 --- a/plugin-server/src/worker/ingestion/process-event.ts +++ b/plugin-server/src/worker/ingestion/process-event.ts @@ -104,6 +104,30 @@ export class EventsProcessor { return result } + private getElementsChain(properties: Properties): string { + /* + We're deprecating $elements in favor of $elements_chain, which doesn't require extra + processing on the ingestion side and is the way we store elements in ClickHouse. + As part of that we'll move posthog-js to send us $elements_chain as string directly, + but we still need to support the old way of sending $elements and converting them + to $elements_chain, while everyone hasn't upgraded. + */ + let elementsChain = '' + if (properties['$elements_chain']) { + elementsChain = properties['$elements_chain'] + } else if (properties['$elements']) { + const elements: Record[] | undefined = properties['$elements'] + let elementsList: Element[] = [] + if (elements && elements.length) { + elementsList = extractElements(elements) + elementsChain = elementsToString(elementsList) + } + } + delete properties['$elements_chain'] + delete properties['$elements'] + return elementsChain + } + private async capture( eventUuid: string, team: Team, @@ -113,13 +137,6 @@ export class EventsProcessor { timestamp: DateTime ): Promise { event = sanitizeEventName(event) - const elements: Record[] | undefined = properties['$elements'] - let elementsList: Element[] = [] - - if (elements && elements.length) { - elementsList = extractElements(elements) - delete properties['$elements'] - } if (properties['$ip'] && team.anonymize_ips) { delete properties['$ip'] @@ -148,7 +165,6 @@ export class EventsProcessor { distinctId, properties, timestamp: timestamp.toISO() as ISOTimestamp, - elementsList, teamId: team.id, } } @@ -168,17 +184,20 @@ export class EventsProcessor { preIngestionEvent: PreIngestionEvent, person: Person ): Promise<[RawClickHouseEvent, Promise]> { - const { - eventUuid: uuid, - event, - teamId, - distinctId, - properties, - timestamp, - elementsList: elements, - } = preIngestionEvent + const { eventUuid: uuid, event, teamId, distinctId, properties, timestamp } = preIngestionEvent - const elementsChain = elements && elements.length ? elementsToString(elements) : '' + let elementsChain = '' + try { + elementsChain = this.getElementsChain(properties) + } catch (error) { + Sentry.captureException(error, { tags: { team_id: teamId } }) + status.warn('⚠️', 'Failed to process elements', { + uuid, + teamId: teamId, + properties, + error, + }) + } const groupIdentifiers = this.getGroupIdentifiers(properties) const groupsColumns = await this.db.getGroupsColumns(teamId, groupIdentifiers) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts index b155e47d7d6f6..4c1467653f324 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts @@ -83,7 +83,6 @@ describe('prepareEventStep()', () => { expect(response).toEqual({ distinctId: 'my_id', - elementsList: [], event: 'default event', eventUuid: '017ef865-19da-0000-3b60-1506093bf40f', properties: { @@ -104,7 +103,6 @@ describe('prepareEventStep()', () => { expect(response).toEqual({ distinctId: 'my_id', - elementsList: [], event: 'default event', eventUuid: '017ef865-19da-0000-3b60-1506093bf40f', properties: {}, @@ -113,4 +111,45 @@ describe('prepareEventStep()', () => { }) expect(hub.db.kafkaProducer!.queueMessage).not.toHaveBeenCalled() }) + + // Tests combo of prepareEvent + createEvent + it('extracts elements_chain from properties', async () => { + const event: PluginEvent = { ...pluginEvent, ip: null, properties: { $elements_chain: 'random string', a: 1 } } + const preppedEvent = await prepareEventStep(runner, event) + const [chEvent, _] = await runner.hub.eventsProcessor.createEvent(preppedEvent, person) + + expect(chEvent.elements_chain).toEqual('random string') + expect(chEvent.properties).toEqual('{"a":1}') + }) + // Tests combo of prepareEvent + createEvent + it('uses elements_chain if both elements and elements_chain are present', async () => { + const event: PluginEvent = { + ...pluginEvent, + ip: null, + properties: { + $elements_chain: 'random string', + a: 1, + $elements: [{ tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: 'text' }], + }, + } + const preppedEvent = await prepareEventStep(runner, event) + const [chEvent, _] = await runner.hub.eventsProcessor.createEvent(preppedEvent, person) + + expect(chEvent.elements_chain).toEqual('random string') + expect(chEvent.properties).toEqual('{"a":1}') + }) + + // Tests combo of prepareEvent + createEvent + it('processes elements correctly', async () => { + const event: PluginEvent = { + ...pluginEvent, + ip: null, + properties: { a: 1, $elements: [{ tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: 'text' }] }, + } + const preppedEvent = await prepareEventStep(runner, event) + const [chEvent, _] = await runner.hub.eventsProcessor.createEvent(preppedEvent, person) + + expect(chEvent.elements_chain).toEqual('div:nth-child="1"nth-of-type="2"text="text"') + expect(chEvent.properties).toEqual('{"a":1}') + }) })