chore: Keep ip only within properties

PostHog · Sep 28, 2023 · 6514987 · 6514987
1 parent 300193c
commit 6514987
Show file tree

Hide file tree

Showing 7 changed files with 14 additions and 24 deletions.
diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts
@@ -651,7 +651,6 @@ export interface ClickHouseEvent extends BaseEvent {
 interface BaseIngestionEvent {
     eventUuid: string
     event: string
-    ip: string | null
     teamId: TeamId
     distinctId: string
     properties: Properties

diff --git a/plugin-server/src/utils/event.ts b/plugin-server/src/utils/event.ts
@@ -15,7 +15,7 @@ import {
 export function convertToProcessedPluginEvent(event: PostIngestionEvent): ProcessedPluginEvent {
     return {
         distinct_id: event.distinctId,
-        ip: event.ip,
+        ip: null, // deprecated : within properties[$ip] now
         team_id: event.teamId,
         event: event.event,
         properties: event.properties,
@@ -67,7 +67,6 @@ export function convertToIngestionEvent(event: RawClickHouseEvent, skipElementsC
     return {
         eventUuid: event.uuid,
         event: event.event!,
-        ip: properties['$ip'],
         teamId: event.team_id,
         distinctId: event.distinct_id,
         properties,
@@ -95,6 +94,10 @@ export function normalizeEvent(event: PluginEvent): PluginEvent {
     if (event['$set_once']) {
         properties['$set_once'] = { ...properties['$set_once'], ...event['$set_once'] }
     }
+    if (!properties['$ip'] && event.ip) {
+        // if $ip wasn't sent with the event, then add what we got from capture
+        properties['$ip'] = event.ip
+    }
     if (!['$snapshot', '$performance_event'].includes(event.event)) {
         properties = personInitialAndUTMProperties(properties)
     }
@@ -113,7 +116,6 @@ export function formPipelineEvent(message: Message): PipelineEvent {
     const event: PipelineEvent = normalizeEvent({
         ...combinedEvent,
         site_url: combinedEvent.site_url || null,
-        ip: combinedEvent.ip || null,
     })
     return event
 }
@@ -122,7 +124,7 @@ export function formPluginEvent(event: RawClickHouseEvent): PluginEvent {
     const postIngestionEvent = convertToIngestionEvent(event)
     return {
         distinct_id: postIngestionEvent.distinctId,
-        ip: postIngestionEvent.properties['$ip'],
+        ip: null, // deprecated : within properties[$ip] now
         site_url: '',
         team_id: postIngestionEvent.teamId,
         now: DateTime.now().toISO(),

diff --git a/plugin-server/src/worker/ingestion/event-pipeline/prepareEventStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/prepareEventStep.ts
@@ -6,7 +6,7 @@ import { captureIngestionWarning } from '../utils'
 import { EventPipelineRunner } from './runner'
 
 export async function prepareEventStep(runner: EventPipelineRunner, event: PluginEvent): Promise<PreIngestionEvent> {
-    const { ip, team_id, uuid } = event
+    const { team_id, uuid } = event
     const invalidTimestampCallback = function (type: string, details: Record<string, any>) {
         // TODO: make that metric name more generic when transitionning to prometheus
         runner.hub.statsd?.increment('process_event_invalid_timestamp', { teamId: String(team_id), type: type })
@@ -15,7 +15,6 @@ export async function prepareEventStep(runner: EventPipelineRunner, event: Plugi
     }
     const preIngestionEvent = await runner.hub.eventsProcessor.processEvent(
         String(event.distinct_id),
-        ip,
         event,
         team_id,
         parseEventTimestamp(event, invalidTimestampCallback),

diff --git a/plugin-server/src/worker/ingestion/process-event.ts b/plugin-server/src/worker/ingestion/process-event.ts
@@ -59,7 +59,6 @@ export class EventsProcessor {
 
     public async processEvent(
         distinctId: string,
-        ip: string | null,
         data: PluginEvent,
         teamId: number,
         timestamp: DateTime,
@@ -90,7 +89,7 @@ export class EventsProcessor {
                 eventUuid,
             })
             try {
-                result = await this.capture(eventUuid, ip, team, data['event'], distinctId, properties, timestamp)
+                result = await this.capture(eventUuid, team, data['event'], distinctId, properties, timestamp)
                 this.pluginsServer.statsd?.timing('kafka_queue.single_save.standard', singleSaveTimer, {
                     team_id: teamId.toString(),
                 })
@@ -105,7 +104,6 @@ export class EventsProcessor {
 
     private async capture(
         eventUuid: string,
-        ip: string | null,
         team: Team,
         event: string,
         distinctId: string,
@@ -121,13 +119,8 @@ export class EventsProcessor {
             delete properties['$elements']
         }
 
-        if (ip) {
-            if (team.anonymize_ips) {
-                ip = null
-                delete properties['$ip']
-            } else if (!('$ip' in properties)) {
-                properties['$ip'] = ip
-            }
+        if (properties['$ip'] && team.anonymize_ips) {
+            delete properties['$ip']
         }
 
         try {
@@ -150,7 +143,6 @@ export class EventsProcessor {
         return {
             eventUuid,
             event,
-            ip,
             distinctId,
             properties,
             timestamp: timestamp.toISO() as ISOTimestamp,

diff --git a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts
@@ -39,7 +39,6 @@ jest.mock('./../../../src/worker/ingestion/utils')
 const event: PostIngestionEvent = {
     eventUuid: 'uuid1',
     distinctId: 'my_id',
-    ip: '127.0.0.1',
     teamId: 2,
     timestamp: '2020-02-23T02:15:00.000Z' as ISOTimestamp,
     event: '$pageview',

diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/prepareEventStep.test.ts
@@ -11,13 +11,15 @@ jest.mock('../../../../src/utils/status')
 
 const pluginEvent: PluginEvent = {
     distinct_id: 'my_id',
-    ip: '127.0.0.1',
+    ip: null,
     site_url: 'http://localhost',
     team_id: 2,
     now: '2020-02-23T02:15:00Z',
     timestamp: '2020-02-23T02:15:00Z',
     event: 'default event',
-    properties: {},
+    properties: {
+        $ip: '127.0.0.1',
+    },
     uuid: '017ef865-19da-0000-3b60-1506093bf40f',
 }
 
@@ -84,7 +86,6 @@ describe('prepareEventStep()', () => {
             elementsList: [],
             event: 'default event',
             eventUuid: '017ef865-19da-0000-3b60-1506093bf40f',
-            ip: '127.0.0.1',
             properties: {
                 $ip: '127.0.0.1',
             },
@@ -106,7 +107,6 @@ describe('prepareEventStep()', () => {
             elementsList: [],
             event: 'default event',
             eventUuid: '017ef865-19da-0000-3b60-1506093bf40f',
-            ip: null,
             properties: {},
             teamId: 2,
             timestamp: '2020-02-23T02:15:00.000Z',

diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/runAsyncHandlersStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/runAsyncHandlersStep.test.ts
@@ -12,7 +12,6 @@ const testElements: any = ['element1', 'element2']
 const ingestionEvent: PostIngestionEvent = {
     eventUuid: 'uuid1',
     distinctId: 'my_id',
-    ip: '127.0.0.1',
     teamId: 2,
     timestamp: '2020-02-23T02:15:00.000Z' as ISOTimestamp,
     event: '$pageview',