Started adding global finding

plugin-server/src/cdp/cdp-api.ts

@@ -121,7 +121,7 @@ export class CdpApi {
             }
 
             // TODO: Type the configuration better so we don't make mistakes here
-            await this.hogFunctionManager.enrichWithIntegrations([compoundConfiguration])
+            await this.hogFunctionManager.prepareHogFunctions([compoundConfiguration])
 
             let response = this.hogExecutor.execute(compoundConfiguration, invocation)
             const logs: HogFunctionLogEntry[] = []

plugin-server/src/cdp/hog-function-manager.ts

@@ -5,6 +5,7 @@ import { PostgresRouter, PostgresUse } from '../utils/db/postgres'
 import { PubSub } from '../utils/pubsub'
 import { status } from '../utils/status'
 import { HogFunctionType, IntegrationType } from './types'
+import { findGlobalsFast } from './utils'
 
 export type HogFunctionMap = Record<HogFunctionType['id'], HogFunctionType>
 export type HogFunctionCache = Record<Team['id'], HogFunctionMap>
@@ -94,7 +95,7 @@ export class HogFunctionManager {
             )
         ).rows
 
-        await this.enrichWithIntegrations(items)
+        await this.prepareHogFunctions(items)
 
         const cache: HogFunctionCache = {}
         for (const item of items) {
@@ -123,7 +124,7 @@ export class HogFunctionManager {
             )
         ).rows
 
-        await this.enrichWithIntegrations(items)
+        await this.prepareHogFunctions(items)
 
         if (!this.cache[teamId]) {
             this.cache[teamId] = {}
@@ -150,7 +151,7 @@ export class HogFunctionManager {
                 'fetchHogFunction'
             )
         ).rows
-        await this.enrichWithIntegrations(items)
+        await this.prepareHogFunctions(items)
         return items[0] ?? null
     }
 
@@ -160,13 +161,44 @@ export class HogFunctionManager {
         const items: HogFunctionType[] = Object.values(this.cache[teamId] || {})
         const itemsToReload = items.filter((item) => ids.some((id) => item.depends_on_integration_ids?.has(id)))
 
-        return this.enrichWithIntegrations(itemsToReload)
+        return this.prepareHogFunctions(itemsToReload)
     }
 
-    public async enrichWithIntegrations(items: HogFunctionType[]): Promise<void> {
+    public async prepareHogFunctions(items: HogFunctionType[]): Promise<void> {
         const integrationIds: number[] = []
 
         items.forEach((item) => {
+            const filtersGlobals = findGlobalsFast(item.filters?.bytecode)
+
+            // Add the hog function globals
+            let functionGlobals = findGlobalsFast(item.bytecode)
+
+            // Extend it with the inputs that are used for the function
+            Object.values(item.inputs || {}).forEach((input) => {
+                functionGlobals = functionGlobals.union(findGlobalsFast(input.bytecode))
+            })
+
+            item.used_globals = {
+                all: functionGlobals.union(filtersGlobals),
+                filters: filtersGlobals,
+                function: functionGlobals,
+            }
+        })
+
+        items.forEach((item) => {
+            // First off we find all globals used in this function - this can later be used for more efficient loading
+            const filtersGlobals = findGlobalsFast(item.filters?.bytecode)
+            let functionGlobals = findGlobalsFast(item.bytecode)
+            Object.values(item.inputs || {}).forEach((input) => {
+                functionGlobals = functionGlobals.union(findGlobalsFast(input.bytecode))
+            })
+
+            item.used_globals = {
+                all: functionGlobals.union(filtersGlobals),
+                filters: filtersGlobals,
+                function: functionGlobals,
+            }
+
             item.inputs_schema?.forEach((schema) => {
                 if (schema.type === 'integration') {
                     const input = item.inputs?.[schema.key]

plugin-server/src/cdp/types.ts

@@ -226,12 +226,17 @@ export type HogFunctionType = {
     inputs?: Record<string, HogFunctionInputType>
     filters?: HogFunctionFilters | null
     depends_on_integration_ids?: Set<IntegrationType['id']>
+    used_globals?: {
+        all: Set<string>
+        filters: Set<string>
+        function: Set<string>
+    }
 }
 
 export type HogFunctionInputType = {
     value: any
     secret?: boolean
-    bytecode?: HogBytecode | object
+    bytecode?: HogBytecode
 }
 
 export type IntegrationType = {

plugin-server/src/cdp/utils.ts

@@ -1,5 +1,6 @@
 // NOTE: PostIngestionEvent is our context event - it should never be sent directly to an output, but rather transformed into a lightweight schema
 
+import { Operation } from '@posthog/hogvm'
 import { DateTime } from 'luxon'
 import { gunzip, gzip } from 'zlib'
 
@@ -158,3 +159,23 @@ export const unGzipObject = async <T extends object>(data: string): Promise<T> =
 
     return JSON.parse(res.toString())
 }
+
+/** Find globals by quickly skimming through the bytecode and looking for a GET_GLOBAL pattern */
+export function findGlobalsFast(bytecode?: any[]): Set<string> {
+    if (!bytecode) {
+        return new Set()
+    }
+    const globals = new Set<string>()
+    for (let i = 2; i < bytecode.length - 1; i++) {
+        if (
+            bytecode[i] === Operation.GET_GLOBAL && // current = GET_GLOBAL
+            typeof bytecode[i - 1] === 'string' && // prev = string
+            bytecode[i - 2] == Operation.STRING && // prev.prev = op.String
+            typeof bytecode[i + 1] === 'number' // next = number
+        ) {
+            globals.add(bytecode[i - 1])
+        }
+    }
+
+    return globals
+}