feat(plugin-server): lazy distinct_id/person row creation (#21670)

* split out InternalPerson and Person types * disable old backfill code * add lazy person env var setting * lazily create person/distinct_id rows * remove backfill.ts * drop properties after person lookup * remove leftover backfill bit
PostHog · Apr 23, 2024 · ddd0a05 · ddd0a05
1 parent ec02c4c
commit ddd0a05
Show file tree

Hide file tree

Showing 12 changed files with 246 additions and 255 deletions.
diff --git a/plugin-server/src/backfill.ts b/plugin-server/src/backfill.ts
diff --git a/plugin-server/src/config/config.ts b/plugin-server/src/config/config.ts
@@ -137,6 +137,7 @@ export function getDefaultConfig(): PluginsServerConfig {
         RUSTY_HOOK_ROLLOUT_PERCENTAGE: 0,
         RUSTY_HOOK_URL: '',
         CAPTURE_CONFIG_REDIS_HOST: null,
+        LAZY_PERSON_CREATION_TEAMS: '',
 
         STARTUP_PROFILE_DURATION_SECONDS: 300, // 5 minutes
         STARTUP_PROFILE_CPU: false,

diff --git a/plugin-server/src/index.ts b/plugin-server/src/index.ts
@@ -1,4 +1,3 @@
-import { startBackfill } from './backfill'
 import { getPluginServerCapabilities } from './capabilities'
 import { defaultConfig } from './config/config'
 import { initApp } from './init'
@@ -51,9 +50,6 @@ switch (alternativeMode) {
             }
         })()
         break
-    case AlternativeMode.Backfill:
-        void startBackfill()
-        break
     default:
         // void the returned promise
         initApp(defaultConfig)

diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts
@@ -210,6 +210,7 @@ export interface PluginsServerConfig {
     SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP: boolean
     PIPELINE_STEP_STALLED_LOG_TIMEOUT: number
     CAPTURE_CONFIG_REDIS_HOST: string | null // Redis cluster to use to coordinate with capture (overflow, routing)
+    LAZY_PERSON_CREATION_TEAMS: string
 
     // dump profiles to disk, covering the first N seconds of runtime
     STARTUP_PROFILE_DURATION_SECONDS: number
@@ -293,6 +294,7 @@ export interface Hub extends PluginsServerConfig {
     pluginConfigsToSkipElementsParsing: ValueMatcher<number>
     poeEmbraceJoinForTeams: ValueMatcher<number>
     poeWritesExcludeTeams: ValueMatcher<number>
+    lazyPersonCreationTeams: ValueMatcher<number>
     // lookups
     eventsToDropByToken: Map<string, string[]>
 }
@@ -733,11 +735,19 @@ export interface RawPerson extends BasePerson {
 }
 
 /** Usable Person model. */
-export interface Person extends BasePerson {
+export interface InternalPerson extends BasePerson {
     created_at: DateTime
     version: number
 }
 
+/** Person model exposed outside of person-specific DB logic. */
+export interface Person {
+    team_id: number
+    properties: Properties
+    uuid: string
+    created_at: DateTime
+}
+
 /** Clickhouse Person model. */
 export interface ClickHousePerson {
     id: string