diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index b9bfe64dce03d6..6346533f42bf4b 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -651,7 +651,6 @@ export interface ClickHouseEvent extends BaseEvent { interface BaseIngestionEvent { eventUuid: string event: string - ip: string | null teamId: TeamId distinctId: string properties: Properties diff --git a/plugin-server/src/utils/event.ts b/plugin-server/src/utils/event.ts index f7417bf6cb3aa9..5d6155d69fc4ef 100644 --- a/plugin-server/src/utils/event.ts +++ b/plugin-server/src/utils/event.ts @@ -15,7 +15,7 @@ import { export function convertToProcessedPluginEvent(event: PostIngestionEvent): ProcessedPluginEvent { return { distinct_id: event.distinctId, - ip: event.ip, + ip: null, // deprecated : within properties[$ip] now team_id: event.teamId, event: event.event, properties: event.properties, @@ -67,7 +67,6 @@ export function convertToIngestionEvent(event: RawClickHouseEvent, skipElementsC return { eventUuid: event.uuid, event: event.event!, - ip: properties['$ip'], teamId: event.team_id, distinctId: event.distinct_id, properties, @@ -95,6 +94,10 @@ export function normalizeEvent(event: PluginEvent): PluginEvent { if (event['$set_once']) { properties['$set_once'] = { ...properties['$set_once'], ...event['$set_once'] } } + if (!properties['$ip'] && event.ip) { + // if $ip wasn't sent with the event, then add what we got from capture + properties['$ip'] = event.ip + } if (!['$snapshot', '$performance_event'].includes(event.event)) { properties = personInitialAndUTMProperties(properties) } @@ -113,7 +116,6 @@ export function formPipelineEvent(message: Message): PipelineEvent { const event: PipelineEvent = normalizeEvent({ ...combinedEvent, site_url: combinedEvent.site_url || null, - ip: combinedEvent.ip || null, }) return event } @@ -122,7 +124,7 @@ export function formPluginEvent(event: RawClickHouseEvent): PluginEvent { const postIngestionEvent = convertToIngestionEvent(event) return { distinct_id: postIngestionEvent.distinctId, - ip: postIngestionEvent.properties['$ip'], + ip: null, // deprecated : within properties[$ip] now site_url: '', team_id: postIngestionEvent.teamId, now: DateTime.now().toISO(), diff --git a/plugin-server/src/worker/ingestion/event-pipeline/prepareEventStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/prepareEventStep.ts index f55c5240b43516..d44bb440b3c4a1 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/prepareEventStep.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/prepareEventStep.ts @@ -6,7 +6,7 @@ import { captureIngestionWarning } from '../utils' import { EventPipelineRunner } from './runner' export async function prepareEventStep(runner: EventPipelineRunner, event: PluginEvent): Promise { - const { ip, team_id, uuid } = event + const { team_id, uuid } = event const invalidTimestampCallback = function (type: string, details: Record) { // TODO: make that metric name more generic when transitionning to prometheus runner.hub.statsd?.increment('process_event_invalid_timestamp', { teamId: String(team_id), type: type }) @@ -15,7 +15,6 @@ export async function prepareEventStep(runner: EventPipelineRunner, event: Plugi } const preIngestionEvent = await runner.hub.eventsProcessor.processEvent( String(event.distinct_id), - ip, event, team_id, parseEventTimestamp(event, invalidTimestampCallback), diff --git a/plugin-server/src/worker/ingestion/process-event.ts b/plugin-server/src/worker/ingestion/process-event.ts index 155404701790e0..9424e50859d48e 100644 --- a/plugin-server/src/worker/ingestion/process-event.ts +++ b/plugin-server/src/worker/ingestion/process-event.ts @@ -59,7 +59,6 @@ export class EventsProcessor { public async processEvent( distinctId: string, - ip: string | null, data: PluginEvent, teamId: number, timestamp: DateTime, @@ -90,7 +89,7 @@ export class EventsProcessor { eventUuid, }) try { - result = await this.capture(eventUuid, ip, team, data['event'], distinctId, properties, timestamp) + result = await this.capture(eventUuid, team, data['event'], distinctId, properties, timestamp) this.pluginsServer.statsd?.timing('kafka_queue.single_save.standard', singleSaveTimer, { team_id: teamId.toString(), }) @@ -105,7 +104,6 @@ export class EventsProcessor { private async capture( eventUuid: string, - ip: string | null, team: Team, event: string, distinctId: string, @@ -121,13 +119,8 @@ export class EventsProcessor { delete properties['$elements'] } - if (ip) { - if (team.anonymize_ips) { - ip = null - delete properties['$ip'] - } else if (!('$ip' in properties)) { - properties['$ip'] = ip - } + if (properties['$ip'] && team.anonymize_ips) { + delete properties['$ip'] } try { @@ -150,7 +143,6 @@ export class EventsProcessor { return { eventUuid, event, - ip, distinctId, properties, timestamp: timestamp.toISO() as ISOTimestamp, diff --git a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts index 0580f53d2724bc..1c419f7cc71d05 100644 --- a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts +++ b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts @@ -39,7 +39,6 @@ jest.mock('./../../../src/worker/ingestion/utils') const event: PostIngestionEvent = { eventUuid: 'uuid1', distinctId: 'my_id', - ip: '127.0.0.1', teamId: 2, timestamp: '2020-02-23T02:15:00.000Z' as ISOTimestamp, event: '$pageview', diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts index b72eea30400c49..b155e47d7d6f65 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts @@ -11,13 +11,15 @@ jest.mock('../../../../src/utils/status') const pluginEvent: PluginEvent = { distinct_id: 'my_id', - ip: '127.0.0.1', + ip: null, site_url: 'http://localhost', team_id: 2, now: '2020-02-23T02:15:00Z', timestamp: '2020-02-23T02:15:00Z', event: 'default event', - properties: {}, + properties: { + $ip: '127.0.0.1', + }, uuid: '017ef865-19da-0000-3b60-1506093bf40f', } @@ -84,7 +86,6 @@ describe('prepareEventStep()', () => { elementsList: [], event: 'default event', eventUuid: '017ef865-19da-0000-3b60-1506093bf40f', - ip: '127.0.0.1', properties: { $ip: '127.0.0.1', }, @@ -106,7 +107,6 @@ describe('prepareEventStep()', () => { elementsList: [], event: 'default event', eventUuid: '017ef865-19da-0000-3b60-1506093bf40f', - ip: null, properties: {}, teamId: 2, timestamp: '2020-02-23T02:15:00.000Z', diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/runAsyncHandlersStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/runAsyncHandlersStep.test.ts index b60892af9f1246..491b57863c1b37 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/runAsyncHandlersStep.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/runAsyncHandlersStep.test.ts @@ -12,7 +12,6 @@ const testElements: any = ['element1', 'element2'] const ingestionEvent: PostIngestionEvent = { eventUuid: 'uuid1', distinctId: 'my_id', - ip: '127.0.0.1', teamId: 2, timestamp: '2020-02-23T02:15:00.000Z' as ISOTimestamp, event: '$pageview',