chore: Keep ip only within properties

plugin-server/src/types.ts

@@ -651,7 +651,6 @@ export interface ClickHouseEvent extends BaseEvent {
 interface BaseIngestionEvent {
     eventUuid: string
     event: string
-    ip: string | null
     teamId: TeamId
     distinctId: string
     properties: Properties

plugin-server/src/utils/event.ts

@@ -15,7 +15,7 @@ import {
 export function convertToProcessedPluginEvent(event: PostIngestionEvent): ProcessedPluginEvent {
     return {
         distinct_id: event.distinctId,
-        ip: event.ip,
+        ip: null, // deprecated : within properties[$ip] now
         team_id: event.teamId,
         event: event.event,
         properties: event.properties,
@@ -67,7 +67,6 @@ export function convertToIngestionEvent(event: RawClickHouseEvent, skipElementsC
     return {
         eventUuid: event.uuid,
         event: event.event!,
-        ip: properties['$ip'],
         teamId: event.team_id,
         distinctId: event.distinct_id,
         properties,
@@ -95,6 +94,13 @@ export function normalizeEvent(event: PluginEvent): PluginEvent {
     if (event['$set_once']) {
         properties['$set_once'] = { ...properties['$set_once'], ...event['$set_once'] }
     }
+    if (!properties['$ip'] && event.ip) {
+        // if $ip wasn't sent with the event, then add what we got from capture
+        properties['$ip'] = event.ip
+    }
+    // For safety while PluginEvent still has an `ip` field
+    event.ip = null
+
     if (!['$snapshot', '$performance_event'].includes(event.event)) {
         properties = personInitialAndUTMProperties(properties)
     }
@@ -113,7 +119,6 @@ export function formPipelineEvent(message: Message): PipelineEvent {
     const event: PipelineEvent = normalizeEvent({
         ...combinedEvent,
         site_url: combinedEvent.site_url || null,
-        ip: combinedEvent.ip || null,
     })
     return event
 }
@@ -122,7 +127,7 @@ export function formPluginEvent(event: RawClickHouseEvent): PluginEvent {
     const postIngestionEvent = convertToIngestionEvent(event)
     return {
         distinct_id: postIngestionEvent.distinctId,
-        ip: postIngestionEvent.properties['$ip'],
+        ip: null, // deprecated : within properties[$ip] now
         site_url: '',
         team_id: postIngestionEvent.teamId,
         now: DateTime.now().toISO(),

plugin-server/src/worker/ingestion/event-pipeline/prepareEventStep.ts

@@ -6,7 +6,7 @@ import { captureIngestionWarning } from '../utils'
 import { EventPipelineRunner } from './runner'
 
 export async function prepareEventStep(runner: EventPipelineRunner, event: PluginEvent): Promise<PreIngestionEvent> {
-    const { ip, team_id, uuid } = event
+    const { team_id, uuid } = event
     const invalidTimestampCallback = function (type: string, details: Record<string, any>) {
         // TODO: make that metric name more generic when transitionning to prometheus
         runner.hub.statsd?.increment('process_event_invalid_timestamp', { teamId: String(team_id), type: type })
@@ -15,7 +15,6 @@ export async function prepareEventStep(runner: EventPipelineRunner, event: Plugi
     }
     const preIngestionEvent = await runner.hub.eventsProcessor.processEvent(
         String(event.distinct_id),
-        ip,
         event,
         team_id,
         parseEventTimestamp(event, invalidTimestampCallback),

plugin-server/src/worker/ingestion/process-event.ts

@@ -59,7 +59,6 @@ export class EventsProcessor {
 
     public async processEvent(
         distinctId: string,
-        ip: string | null,
         data: PluginEvent,
         teamId: number,
         timestamp: DateTime,
@@ -90,7 +89,7 @@ export class EventsProcessor {
                 eventUuid,
             })
             try {
-                result = await this.capture(eventUuid, ip, team, data['event'], distinctId, properties, timestamp)
+                result = await this.capture(eventUuid, team, data['event'], distinctId, properties, timestamp)
                 this.pluginsServer.statsd?.timing('kafka_queue.single_save.standard', singleSaveTimer, {
                     team_id: teamId.toString(),
                 })
@@ -105,7 +104,6 @@ export class EventsProcessor {
 
     private async capture(
         eventUuid: string,
-        ip: string | null,
         team: Team,
         event: string,
         distinctId: string,
@@ -121,13 +119,8 @@ export class EventsProcessor {
             delete properties['$elements']
         }
 
-        if (ip) {
-            if (team.anonymize_ips) {
-                ip = null
-                delete properties['$ip']
-            } else if (!('$ip' in properties)) {
-                properties['$ip'] = ip
-            }
+        if (properties['$ip'] && team.anonymize_ips) {
+            delete properties['$ip']
         }
 
         try {
@@ -150,7 +143,6 @@ export class EventsProcessor {
         return {
             eventUuid,
             event,
-            ip,
             distinctId,
             properties,
             timestamp: timestamp.toISO() as ISOTimestamp,

plugin-server/tests/main/ingestion-queues/each-batch.test.ts

@@ -39,7 +39,6 @@ jest.mock('./../../../src/worker/ingestion/utils')
 const event: PostIngestionEvent = {
     eventUuid: 'uuid1',
     distinctId: 'my_id',
-    ip: '127.0.0.1',
     teamId: 2,
     timestamp: '2020-02-23T02:15:00.000Z' as ISOTimestamp,
     event: '$pageview',

plugin-server/tests/main/process-event.test.ts

@@ -834,7 +834,6 @@ test('capture bad team', async () => {
     await expect(
         eventsProcessor.processEvent(
             'asdfasdfasdf',
-            '',
             {
                 event: '$pageview',
                 properties: { distinct_id: 'asdfasdfasdf', token: team.api_token },
@@ -2367,7 +2366,6 @@ test('throws with bad uuid', async () => {
     await expect(
         eventsProcessor.processEvent(
             'xxx',
-            '',
             { event: 'E', properties: { price: 299.99, name: 'AirPods Pro' } } as any as PluginEvent,
             team.id,
             DateTime.utc(),
@@ -2378,7 +2376,6 @@ test('throws with bad uuid', async () => {
     await expect(
         eventsProcessor.processEvent(
             'xxx',
-            '',
             { event: 'E', properties: { price: 299.99, name: 'AirPods Pro' } } as any as PluginEvent,
             team.id,
             DateTime.utc(),

plugin-server/tests/worker/dead-letter-queue.test.ts

@@ -75,12 +75,11 @@ describe('events dead letter queue', () => {
 
         const dlqEvent = deadLetterQueueEvents[0]
         expect(dlqEvent.event).toEqual('default event')
-        expect(dlqEvent.ip).toEqual('127.0.0.1')
         expect(dlqEvent.team_id).toEqual(2)
         expect(dlqEvent.team_id).toEqual(2)
         expect(dlqEvent.error_location).toEqual('plugin_server_ingest_event:prepareEventStep')
         expect(dlqEvent.error).toEqual('ingestEvent failed. Error: database unavailable')
-        expect(dlqEvent.properties).toEqual(JSON.stringify({ key: 'value' }))
+        expect(dlqEvent.properties).toEqual(JSON.stringify({ key: 'value', $ip: '127.0.0.1' }))
         expect(dlqEvent.event_uuid).toEqual(EVENT_UUID)
     })
 })

plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts

@@ -11,13 +11,15 @@ jest.mock('../../../../src/utils/status')
 
 const pluginEvent: PluginEvent = {
     distinct_id: 'my_id',
-    ip: '127.0.0.1',
+    ip: null,
     site_url: 'http://localhost',
     team_id: 2,
     now: '2020-02-23T02:15:00Z',
     timestamp: '2020-02-23T02:15:00Z',
     event: 'default event',
-    properties: {},
+    properties: {
+        $ip: '127.0.0.1',
+    },
     uuid: '017ef865-19da-0000-3b60-1506093bf40f',
 }
 
@@ -84,7 +86,6 @@ describe('prepareEventStep()', () => {
             elementsList: [],
             event: 'default event',
             eventUuid: '017ef865-19da-0000-3b60-1506093bf40f',
-            ip: '127.0.0.1',
             properties: {
                 $ip: '127.0.0.1',
             },
@@ -106,7 +107,6 @@ describe('prepareEventStep()', () => {
             elementsList: [],
             event: 'default event',
             eventUuid: '017ef865-19da-0000-3b60-1506093bf40f',
-            ip: null,
             properties: {},
             teamId: 2,
             timestamp: '2020-02-23T02:15:00.000Z',

plugin-server/tests/worker/ingestion/event-pipeline/processPersonsStep.test.ts

@@ -29,7 +29,7 @@ describe.each([[true], [false]])('processPersonsStep()', (poEEmbraceJoin) => {
 
         pluginEvent = {
             distinct_id: 'my_id',
-            ip: '127.0.0.1',
+            ip: null,
             site_url: 'http://localhost',
             team_id: teamId,
             now: '2020-02-23T02:15:00Z',

plugin-server/tests/worker/ingestion/event-pipeline/runAsyncHandlersStep.test.ts

@@ -12,7 +12,6 @@ const testElements: any = ['element1', 'element2']
 const ingestionEvent: PostIngestionEvent = {
     eventUuid: 'uuid1',
     distinctId: 'my_id',
-    ip: '127.0.0.1',
     teamId: 2,
     timestamp: '2020-02-23T02:15:00.000Z' as ISOTimestamp,
     event: '$pageview',