chore(devex): hogvm into common folder

.dockerignore

@@ -33,7 +33,7 @@
 !plugin-server/.eslintrc.js
 !plugin-server/.prettierrc
 !share/GeoLite2-City.mmdb
-!hogvm/python
+!common/hogvm/python
 !unit.json.tpl
 !common/plugin_transpiler/src
 !common/plugin_transpiler/*.*

.eslintignore

@@ -3,3 +3,8 @@ jest.config.ts
 node_modules
 rust
 livestream
+common/hogvm/__tests__/__snapshots__
+common/hogvm/typescript/dist
+common/hogvm/typescript/jest.config.js
+common/plugin_transpiler/dist
+common/plugin_transpiler/build.mjs

.github/actions/run-backend-tests/action.yml

@@ -164,7 +164,7 @@ runs:
                   || 'posthog'
               }} ${{ inputs.person-on-events == 'true' && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations" \
                   --ignore=posthog/temporal \
-                  --ignore=hogvm/python/test \
+                  --ignore=common/hogvm/python/test \
                   --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \
                   --durations=100 --durations-min=1.0 --store-durations \
                   $PYTEST_ARGS

.github/workflows/ci-backend.yml

@@ -68,7 +68,7 @@ jobs:
                         # including the negated rule appears to work
                         # but makes it always match because the checked file always isn't `ee/frontend/**` 🙈
                         - 'ee/**/*'
-                        - 'hogvm/**/*'
+                        - 'common/hogvm/**/*'
                         - 'posthog/**/*'
                         - 'bin/*.py'
                         - requirements.txt

.github/workflows/ci-hog.yml

@@ -37,7 +37,7 @@ jobs:
                   filters: |
                       hog:
                         # Avoid running tests for irrelevant changes
-                        - 'hogvm/**/*'
+                        - 'common/hogvm/**/*'
                         - 'posthog/hogql/**/*'
                         - 'bin/hog'
                         - 'bin/hoge'
@@ -116,22 +116,22 @@ jobs:
 
             - name: Check if STL bytecode is up to date
               run: |
-                  python -m hogvm.stl.compile
+                  python -m common.hogvm.stl.compile
                   git diff --exit-code
 
             - name: Run HogVM Python tests
               run: |
-                  pytest hogvm
+                  pytest common/hogvm
 
             - name: Run HogVM TypeScript tests
               run: |
-                  cd hogvm/typescript
+                  cd common/hogvm/typescript
                   pnpm install --frozen-lockfile
                   pnpm run test
 
             - name: Run Hog tests
               run: |
-                  cd hogvm/typescript
+                  cd common/hogvm/typescript
                   pnpm run build 
                   cd ../
                   ./test.sh && git diff --exit-code
@@ -152,7 +152,7 @@ jobs:
               id: check-package-version
               uses: PostHog/check-package-version@v2
               with:
-                  path: hogvm/typescript
+                  path: common/hogvm/typescript
 
     release-hogvm:
         name: Release new HogVM TypeScript version
@@ -191,9 +191,9 @@ jobs:
                   node-version: 18
                   registry-url: https://registry.npmjs.org
             - name: Install package.json dependencies
-              run: cd hogvm/typescript && pnpm install
+              run: cd common/hogvm/typescript && pnpm install
             - name: Publish the package in the npm registry
-              run: cd hogvm/typescript && npm publish --access public
+              run: cd common/hogvm/typescript && npm publish --access public
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
             - name: Sleep 60 seconds to allow npm to update the package
@@ -225,7 +225,7 @@ jobs:
               id: check-mismatch
               run: |
                   # Extract committed version
-                  HOGVM_VERSION=$(jq -r '.version' hogvm/typescript/package.json)
+                  HOGVM_VERSION=$(jq -r '.version' common/hogvm/typescript/package.json)
 
                   # Compare dependencies in package.json
                   MAIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' package.json | tr -d '^')
@@ -245,7 +245,7 @@ jobs:
             - name: Update package.json versions
               if: env.mismatch == 'true'
               run: |
-                  VERSION=$(jq ".version" hogvm/typescript/package.json -r)
+                  VERSION=$(jq ".version" common/hogvm/typescript/package.json -r)
 
                   retry_pnpm_install() {
                       local retries=0
@@ -283,7 +283,7 @@ jobs:
               if: env.mismatch == 'true'
               uses: EndBug/add-and-commit@v9
               with:
-                  add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "hogvm/typescript/package.json"]'
+                  add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "common/hogvm/typescript/package.json"]'
                   message: 'Update @posthog/hogvm version in package.json'
                   default_author: github_actions
                   github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.gitignore

@@ -56,8 +56,8 @@ gen/
 # Local files generated for the ANTLR VS Code extension (https://github.com/mike-lischke/vscode-antlr4)
 .antlr
 upgrade/
-hogvm/typescript/dist
-hogvm/typescript/.parcel-cache
+common/hogvm/typescript/dist
+common/hogvm/typescript/.parcel-cache
 .wokeignore
 common/plugin_transpiler/dist
 *-esbuild-meta.json

.prettierignore

@@ -17,6 +17,6 @@ dist/
 node_modules/
 pnpm-lock.yaml
 posthog/templates/email/*
-hogvm/typescript/src/stl/bytecode.ts
+common/hogvm/typescript/src/stl/bytecode.ts
 rust/
 livestream/

.test_durations

@@ -4765,9 +4765,9 @@
   "posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py::TestFunnelTrends::test_month_interval": 0.8332772129999739,
   "posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py::TestFunnelTrends::test_no_event_in_period": 0.6817592790000617,
   "posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py::TestFunnelTrends::test_one_person_in_multiple_periods_and_windows": 2.9724146470000505,
-  "hogvm/python/test/test_execute.py::TestBytecodeExecute::test_bytecode_create": 24.460313353000004,
-  "hogvm/python/test/test_execute.py::TestBytecodeExecute::test_errors": 0.003571909999948275,
-  "hogvm/python/test/test_execute.py::TestBytecodeExecute::test_nested_value": 0.004185780000057093,
+  "common/hogvm/python/test/test_execute.py::TestBytecodeExecute::test_bytecode_create": 24.460313353000004,
+  "common/hogvm/python/test/test_execute.py::TestBytecodeExecute::test_errors": 0.003571909999948275,
+  "common/hogvm/python/test/test_execute.py::TestBytecodeExecute::test_nested_value": 0.004185780000057093,
   "posthog/api/dashboards/test/test_dashboard_templates.py::TestDashboardTemplates::test_cannot_escape_team_when_filtering_template_list": 3.904341852000016,
   "posthog/api/dashboards/test/test_dashboard_templates.py::TestDashboardTemplates::test_cant_make_templates_without_teamid_private": 0.044227177000038864,
   "posthog/api/dashboards/test/test_dashboard_templates.py::TestDashboardTemplates::test_create_and_get_dashboard_template_with_tile": 0.03463513100007276,

Dockerfile

@@ -52,7 +52,7 @@
 # Compile and install Node.js dependencies.
 COPY ./plugin-server/package.json ./plugin-server/pnpm-lock.yaml ./plugin-server/tsconfig.json ./
 COPY ./plugin-server/patches/ ./patches/
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     "make" \
     "g++" \
@@ -97,7 +97,7 @@
 # We install those dependencies on a custom folder that we will
 # then copy to the last image.
 COPY requirements.txt ./
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     "build-essential" \
     "git" \
@@ -116,7 +116,7 @@
 
 # Add in Django deps and generate Django's static files.
 COPY manage.py manage.py
-COPY hogvm hogvm/
+COPY common/hogvm common/hogvm/
 COPY posthog posthog/
 COPY products/ products/
 COPY ee ee/
@@ -132,7 +132,7 @@
 SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
 
 # Fetch the GeoLite2-City database that will be used for IP geolocation within Django.
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     "ca-certificates" \
     "curl" \
@@ -155,7 +155,7 @@
 
 # Install OS runtime dependencies.
 # Note: please add in this stage runtime dependences only!
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     "chromium" \
     "chromium-driver" \
@@ -168,11 +168,11 @@
 # Install MS SQL dependencies
 RUN curl https://packages.microsoft.com/keys/microsoft.asc | tee /etc/apt/trusted.gpg.d/microsoft.asc
 RUN curl https://packages.microsoft.com/config/debian/11/prod.list | tee /etc/apt/sources.list.d/mssql-release.list
 RUN apt-get update
 RUN ACCEPT_EULA=Y apt-get install -y msodbcsql18
 
 # Install NodeJS 18.
 RUN apt-get install -y --no-install-recommends \
     "curl" \
     && \
     curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
@@ -216,7 +216,7 @@
 COPY --chown=posthog:posthog manage.py manage.py
 COPY --chown=posthog:posthog posthog posthog/
 COPY --chown=posthog:posthog ee ee/
-COPY --chown=posthog:posthog hogvm hogvm/
+COPY --chown=posthog:posthog common/hogvm common/hogvm/
 COPY --chown=posthog:posthog dags dags/
 COPY --chown=posthog:posthog products products/
 
@@ -235,4 +235,4 @@
 # Expose the port from which we serve OpenMetrics data.
 EXPOSE 8001
 COPY unit.json.tpl /docker-entrypoint.d/unit.json.tpl
 USER root

bin/hog

@@ -3,16 +3,16 @@ set -e
 
 if [[ "$@" == *".hoge"* ]]; then
   if [[ "$@" == *"--python"* ]]; then
-    exec python3 -m hogvm.python.cli "$@"
+    exec python3 -m common.hogvm.python.cli "$@"
 
   elif [[ "$@" == *"--debug"* ]]; then
     echo "Running in Python VM with debug mode"
-    exec python3 -m hogvm.python.cli "$@"
+    exec python3 -m common.hogvm.python.cli "$@"
 
   else # default to nodejs because it's fastest
     set -- "${@/--nodejs/}"
-    VM_PATH="$(dirname "$0")/../hogvm/typescript"
-    CLI_PATH="$(dirname "$0")/../hogvm/typescript/dist/cli.js"
+    VM_PATH="$(dirname "$0")/../common/hogvm/typescript"
+    CLI_PATH="$(dirname "$0")/../common/hogvm/typescript/dist/cli.js"
     # TODO: recompile if something changed
     if [ ! -f $CLI_PATH ]; then
       echo "Building TypeScript HogVM"

bin/tests

@@ -37,4 +37,4 @@ else
     TEST_CONCURRENCY="-n $XDIST_WORKERS"
 fi
 
-nodemon -w ./posthog -w ./hogvm/python -w ./ee --ext py --exec "OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES pytest --reuse-db --durations-min=2.0 ${MIGRATIONS} ${TEST_CONCURRENCY} -s $* --snapshot-update; mypy -p posthog | mypy-baseline filter"
+nodemon -w ./posthog -w ./common/hogvm/python -w ./ee --ext py --exec "OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES pytest --reuse-db --durations-min=2.0 ${MIGRATIONS} ${TEST_CONCURRENCY} -s $* --snapshot-update; mypy -p posthog | mypy-baseline filter"

hogvm/python/debugger.py → common/hogvm/python/debugger.py

@@ -2,8 +2,8 @@
 from time import sleep
 from typing import Any
 
-from hogvm.python.objects import CallFrame
-from hogvm.python.operation import Operation
+from common.hogvm.python.objects import CallFrame
+from common.hogvm.python.operation import Operation
 
 debug_speed = -1
 

hogvm/python/execute.py → common/hogvm/python/execute.py

@@ -5,14 +5,21 @@
 from typing import Any, Optional, TYPE_CHECKING
 from collections.abc import Callable
 
-from hogvm.python.debugger import debugger, color_bytecode
-from hogvm.python.objects import is_hog_error, new_hog_closure, CallFrame, ThrowFrame, new_hog_callable, is_hog_upvalue
-from hogvm.python.operation import Operation, HOGQL_BYTECODE_IDENTIFIER, HOGQL_BYTECODE_IDENTIFIER_V0
-from hogvm.python.stl import STL
-from hogvm.python.stl.bytecode import BYTECODE_STL
+from common.hogvm.python.debugger import debugger, color_bytecode
+from common.hogvm.python.objects import (
+    is_hog_error,
+    new_hog_closure,
+    CallFrame,
+    ThrowFrame,
+    new_hog_callable,
+    is_hog_upvalue,
+)
+from common.hogvm.python.operation import Operation, HOGQL_BYTECODE_IDENTIFIER, HOGQL_BYTECODE_IDENTIFIER_V0
+from common.hogvm.python.stl import STL
+from common.hogvm.python.stl.bytecode import BYTECODE_STL
 from dataclasses import dataclass
 
-from hogvm.python.utils import (
+from common.hogvm.python.utils import (
     UncaughtHogVMException,
     HogVMException,
     get_nested_value,

hogvm/python/stl/bytecode.py → common/hogvm/python/stl/bytecode.py

@@ -1,4 +1,4 @@
-# This file is generated by hogvm/stl/compile.py
+# This file is generated by common/hogvm/stl/compile.py
 # fmt: off
 BYTECODE_STL: dict[str, tuple[list[str], list]] = {
   "arrayCount": (["func", "arr"], [33, 0, 36, 1, 36, 3, 2, "values", 1, 33, 1, 36, 4, 2, "length", 1, 31, 36, 6, 36, 5, 16, 40, 31, 36, 4, 36, 5, 45, 37, 7, 36, 7, 36, 0, 54, 1, 40, 7, 33, 1, 36, 2, 6, 37, 2, 36, 5, 33, 1, 6, 37, 5, 39, -38, 35, 35, 35, 35, 35, 36, 2, 38, 35]),

hogvm/python/stl/date.py → common/hogvm/python/stl/date.py

@@ -3,7 +3,7 @@
 
 import pytz
 
-from hogvm.python.objects import is_hog_date, is_hog_datetime
+from common.hogvm.python.objects import is_hog_date, is_hog_datetime
 
 
 def to_hog_date(year: int, month: int, day: int):

hogvm/python/stl/print.py → common/hogvm/python/stl/print.py

@@ -1,6 +1,6 @@
 import re
 
-from hogvm.python.objects import is_hog_datetime, is_hog_date, is_hog_error, is_hog_closure, is_hog_callable
+from common.hogvm.python.objects import is_hog_datetime, is_hog_date, is_hog_error, is_hog_closure, is_hog_callable
 
 # Copied from clickhouse_driver.util.escape, adapted only from single quotes to backquotes.
 escape_chars_map = {

hogvm/python/test/test_execute.py → common/hogvm/python/test/test_execute.py

@@ -3,13 +3,13 @@
 from collections.abc import Callable
 
 
-from hogvm.python.execute import execute_bytecode, get_nested_value
-from hogvm.python.operation import (
+from common.hogvm.python.execute import execute_bytecode, get_nested_value
+from common.hogvm.python.operation import (
     Operation as op,
     HOGQL_BYTECODE_IDENTIFIER as _H,
     HOGQL_BYTECODE_VERSION as VERSION,
 )
-from hogvm.python.utils import UncaughtHogVMException
+from common.hogvm.python.utils import UncaughtHogVMException
 from posthog.hogql.compiler.bytecode import create_bytecode
 from posthog.hogql.parser import parse_expr, parse_program
 

hogvm/stl/compile.py → common/hogvm/stl/compile.py

@@ -1,15 +1,15 @@
 # Run from project root (cd ../..)
-# python3 -m hogvm.stl.compile
+# python3 -m common.hogvm.stl.compile
 
 import glob
 import json
 
 from posthog.hogql import ast
 from posthog.hogql.compiler.bytecode import create_bytecode, parse_program
 
-source = "hogvm/stl/src/*.hog"
-target_ts = "hogvm/typescript/src/stl/bytecode.ts"
-target_py = "hogvm/python/stl/bytecode.py"
+source = "common/hogvm/stl/src/*.hog"
+target_ts = "common/hogvm/typescript/src/stl/bytecode.ts"
+target_py = "common/hogvm/python/stl/bytecode.py"
 
 bytecodes: dict[str, [list[str], list[any]]] = {}
 
@@ -29,14 +29,14 @@
         exit(1)
 
 with open(target_ts, "w") as output:
-    output.write("// This file is generated by hogvm/stl/compile.py\n")
+    output.write("// This file is generated by common/hogvm/stl/compile.py\n")
     output.write("export const BYTECODE_STL: Record<string, [string[], any[]]> = {\n")
     for name, (params, bytecode) in sorted(bytecodes.items()):
         output.write(f'  "{name}": [{json.dumps(params)}, {json.dumps(bytecode)}],\n')
     output.write("}\n")
 
 with open(target_py, "w") as output:
-    output.write("# This file is generated by hogvm/stl/compile.py\n")
+    output.write("# This file is generated by common/hogvm/stl/compile.py\n")
     output.write("# fmt: off\n")
     output.write("BYTECODE_STL: dict[str, tuple[list[str], list]] = {\n")
     for name, (params, bytecode) in sorted(bytecodes.items()):

hogvm/test.sh → common/hogvm/test.sh

@@ -12,15 +12,15 @@ cd typescript
 pnpm run build
 cd ..
 
-# Navigate to the project root (parent directory of 'hogvm')
-cd ..
+# Navigate to the project root (parent of 'common/hogvm')
+cd ../..
 
 # Function to compute the basename for a given file
 get_basename() {
     local file="$1"
     local base="${file%.hog}"
     base="${base##*/}"
-    echo "hogvm/__tests__/__snapshots__/$base"
+    echo "common/hogvm/__tests__/__snapshots__/$base"
 }
 
 # Function to check if a value is in an array
@@ -39,9 +39,9 @@ is_in_array() {
 # Check if an argument is provided
 if [ "$#" -eq 1 ]; then
     test_file="$1"
-    # Adjust the test file path if it doesn't start with 'hogvm/'
-    if [[ ! "$test_file" == hogvm/* ]]; then
-        test_file="hogvm/__tests__/$test_file"
+    # Adjust the test file path if it doesn't start with 'common/hogvm/'
+    if [[ ! "$test_file" == common/hogvm/* ]]; then
+        test_file="common/hogvm/__tests__/$test_file"
     fi
     # Check if the test file exists
     if [ ! -f "$test_file" ]; then
@@ -54,18 +54,18 @@ if [ "$#" -eq 1 ]; then
     rm -f "$basename.stdout.nodejs" "$basename.stdout.python" "$basename.stdout.compiledjs"
 else
     shopt -s nullglob
-    test_files=(hogvm/__tests__/*.hog)
+    test_files=(common/hogvm/__tests__/*.hog)
     shopt -u nullglob
 
     if [ ${#test_files[@]} -eq 0 ]; then
-        echo "No test files found in hogvm/__tests__/"
+        echo "No test files found in common/hogvm/__tests__/"
         exit 1
     fi
 
     # Remove all previous outputs
-    rm -f hogvm/__tests__/__snapshots__/*.stdout.nodejs
-    rm -f hogvm/__tests__/__snapshots__/*.stdout.python
-    rm -f hogvm/__tests__/__snapshots__/*.stdout.compiledjs
+    rm -f common/hogvm/__tests__/__snapshots__/*.stdout.nodejs
+    rm -f common/hogvm/__tests__/__snapshots__/*.stdout.python
+    rm -f common/hogvm/__tests__/__snapshots__/*.stdout.compiledjs
 fi
 
 for file in "${test_files[@]}"; do

hogvm/typescript/package.json → common/hogvm/typescript/package.json

@@ -11,7 +11,7 @@
         "test": "jest --runInBand --forceExit",
         "clean": "rm -rf dist .parcel-cache",
         "build": "pnpm clean && pnpm build:stl && pnpm build:parcel && pnpm build:cli",
-        "build:stl": "cd ../.. && python3 -m hogvm.stl.compile",
+        "build:stl": "cd ../../.. && python3 -m common.hogvm.stl.compile",
         "build:parcel": "parcel build",
         "build:compile": "tsc -p tsconfig.build.json",
         "build:cli": "cp ./src/cli.js ./dist/cli.js",