From e9e34417210f509983229f6ad8d8c37924a14448 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 27 Aug 2024 09:45:37 +0200 Subject: [PATCH] feat(hogvm): serialize the stack between async operations (#24574) --- hogvm/typescript/package.json | 2 +- .../typescript/src/__tests__/execute.test.ts | 22 +++++++++++++++++++ hogvm/typescript/src/execute.ts | 10 ++++----- plugin-server/package.json | 2 +- plugin-server/pnpm-lock.yaml | 9 ++++---- plugin-server/src/cdp/cdp-api.ts | 5 ++--- plugin-server/src/cdp/hog-executor.ts | 4 ++-- 7 files changed, 38 insertions(+), 16 deletions(-) diff --git a/hogvm/typescript/package.json b/hogvm/typescript/package.json index aff5560bc6973..b6e47b51cbcbc 100644 --- a/hogvm/typescript/package.json +++ b/hogvm/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@posthog/hogvm", - "version": "1.0.32", + "version": "1.0.35", "description": "PostHog Hog Virtual Machine", "types": "dist/index.d.ts", "main": "dist/index.js", diff --git a/hogvm/typescript/src/__tests__/execute.test.ts b/hogvm/typescript/src/__tests__/execute.test.ts index 435b7d873abfe..7de7f42d2d3a2 100644 --- a/hogvm/typescript/src/__tests__/execute.test.ts +++ b/hogvm/typescript/src/__tests__/execute.test.ts @@ -1863,4 +1863,26 @@ describe('hogvm execute', () => { new UncaughtHogVMException('RetryError', 'Not a good day', { key: 'value' }) ) }) + + test('returns serialized state', () => { + const bytecode = ['_h', op.STRING, 'key', op.STRING, 'value', op.DICT, 1, op.GET_LOCAL, 0, op.CALL, 'fetch', 1] + expect(exec(bytecode, { asyncFunctions: { fetch: async () => null } })).toEqual({ + asyncFunctionArgs: [{ key: 'value' }], // not a Map + asyncFunctionName: 'fetch', + finished: false, + result: undefined, + state: { + asyncSteps: 1, + bytecode: bytecode, + callStack: [], + declaredFunctions: {}, + ip: 12, + maxMemUsed: 64, + ops: 5, + stack: [{ key: 'value' }], // is not a Map + syncDuration: 0, + throwStack: [], + }, + }) + }) }) diff --git a/hogvm/typescript/src/execute.ts b/hogvm/typescript/src/execute.ts index 389e1ded547f2..dd7865b77f0af 100644 --- a/hogvm/typescript/src/execute.ts +++ b/hogvm/typescript/src/execute.ts @@ -84,10 +84,10 @@ export async function execAsync(bytecode: any[], options?: ExecOptions): Promise const result = await options?.asyncFunctions[response.asyncFunctionName]( ...response.asyncFunctionArgs.map(convertHogToJS) ) - vmState.stack.push(convertJSToHog(result)) + vmState.stack.push(result) } else if (response.asyncFunctionName in ASYNC_STL) { const result = await ASYNC_STL[response.asyncFunctionName]( - response.asyncFunctionArgs, + response.asyncFunctionArgs.map(convertHogToJS), response.asyncFunctionName, options?.timeout ?? DEFAULT_TIMEOUT_MS ) @@ -123,7 +123,7 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { const asyncSteps = vmState ? vmState.asyncSteps : 0 const syncDuration = vmState ? vmState.syncDuration : 0 - const stack: any[] = vmState ? vmState.stack : [] + const stack: any[] = vmState ? vmState.stack.map(convertJSToHog) : [] const memStack: number[] = stack.map((s) => calculateCost(s)) const callStack: [number, number, number][] = vmState ? vmState.callStack : [] const throwStack: [number, number, number][] = vmState ? vmState.throwStack : [] @@ -436,10 +436,10 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { result: undefined, finished: false, asyncFunctionName: name, - asyncFunctionArgs: args, + asyncFunctionArgs: args.map(convertHogToJS), state: { bytecode, - stack, + stack: stack.map(convertHogToJS), callStack, throwStack, declaredFunctions, diff --git a/plugin-server/package.json b/plugin-server/package.json index e5442bf51b68d..f7672084d42eb 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -52,7 +52,7 @@ "@google-cloud/storage": "^5.8.5", "@maxmind/geoip2-node": "^3.4.0", "@posthog/clickhouse": "^1.7.0", - "@posthog/hogvm": "^1.0.32", + "@posthog/hogvm": "^1.0.35", "@posthog/plugin-scaffold": "1.4.4", "@sentry/node": "^7.49.0", "@sentry/profiling-node": "^0.3.0", diff --git a/plugin-server/pnpm-lock.yaml b/plugin-server/pnpm-lock.yaml index f242e25145a74..0bb579b69c438 100644 --- a/plugin-server/pnpm-lock.yaml +++ b/plugin-server/pnpm-lock.yaml @@ -47,8 +47,8 @@ dependencies: specifier: file:../rust/cyclotron-node version: file:../rust/cyclotron-node '@posthog/hogvm': - specifier: ^1.0.32 - version: 1.0.32(luxon@3.4.4)(re2@1.20.3) + specifier: ^1.0.35 + version: 1.0.35(luxon@3.4.4)(re2@1.20.3) '@posthog/plugin-scaffold': specifier: 1.4.4 version: 1.4.4 @@ -3116,8 +3116,8 @@ packages: engines: {node: '>=12'} dev: false - /@posthog/hogvm@1.0.32(luxon@3.4.4)(re2@1.20.3): - resolution: {integrity: sha512-OjgSzs4fZ1Q0KEiON34/dH9TybfVfallANcgoRiNhUd9KstSm55Ds5cpK6HjGMfRRPpPULuDQ77RH3DBjJ2CCA==} + /@posthog/hogvm@1.0.35(luxon@3.4.4)(re2@1.20.3): + resolution: {integrity: sha512-Uq3Cpg0CGxwScdgGddfzDCs42opILFhgnM6wqly55PAZGBhXWxVHgVpejLiWLtG1z8aIMLC8HCbQx1rZheCGMQ==} peerDependencies: luxon: ^3.4.4 re2: ^1.21.3 @@ -10738,4 +10738,5 @@ packages: file:../rust/cyclotron-node: resolution: {directory: ../rust/cyclotron-node, type: directory} name: '@posthog/cyclotron' + version: 0.1.0 dev: false diff --git a/plugin-server/src/cdp/cdp-api.ts b/plugin-server/src/cdp/cdp-api.ts index 553e380e16cdf..943091af13814 100644 --- a/plugin-server/src/cdp/cdp-api.ts +++ b/plugin-server/src/cdp/cdp-api.ts @@ -1,4 +1,3 @@ -import { convertJSToHog } from '@posthog/hogvm' import express from 'express' import { DateTime } from 'luxon' @@ -147,7 +146,7 @@ export class CdpApi { }) // Add the state, simulating what executeAsyncResponse would do - invocation.vmState!.stack.push(convertJSToHog({ status: 200, body: {} })) + invocation.vmState!.stack.push({ status: 200, body: {} }) } else { const asyncInvocationRequest: HogFunctionInvocationAsyncRequest = { state: '', // WE don't care about the state for this level of testing @@ -166,7 +165,7 @@ export class CdpApi { message: 'Failed to execute async function', }) } - invocation.vmState!.stack.push(convertJSToHog(asyncRes?.asyncFunctionResponse.response ?? null)) + invocation.vmState!.stack.push(asyncRes?.asyncFunctionResponse.response ?? null) } logs.push(...response.logs) diff --git a/plugin-server/src/cdp/hog-executor.ts b/plugin-server/src/cdp/hog-executor.ts index 6b590994000c4..3307a9136c7c1 100644 --- a/plugin-server/src/cdp/hog-executor.ts +++ b/plugin-server/src/cdp/hog-executor.ts @@ -1,4 +1,4 @@ -import { calculateCost, convertHogToJS, convertJSToHog, exec, ExecResult } from '@posthog/hogvm' +import { calculateCost, convertHogToJS, exec, ExecResult } from '@posthog/hogvm' import { DateTime } from 'luxon' import { Histogram } from 'prom-client' @@ -217,7 +217,7 @@ export class HogExecutor { } // Add the response to the stack to continue execution - invocation.vmState.stack.push(convertJSToHog(response)) + invocation.vmState.stack.push(response) invocation.timings.push(...timings) const res = this.execute(hogFunction, invocation)