Skip to content

feat(experiments): Auto-join persons table to dw table #22544

feat(experiments): Auto-join persons table to dw table

feat(experiments): Auto-join persons table to dw table #22544

Workflow file for this run

# This workflow runs all of our backend django tests.
#
# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
# .github/workflows/ci-backend-update-test-timing.yml action
name: Hog CI
on:
push:
branches:
- master
paths-ignore:
- rust/**
- livestream/**
pull_request:
paths-ignore:
- rust/**
- livestream/**
jobs:
# Job to decide if we should run backend ci
# See https://github.com/dorny/paths-filter#conditional-execution for more details
changes:
runs-on: ubuntu-24.04
timeout-minutes: 5
name: Determine need to run Hog checks
# Set job outputs to values from filter step
outputs:
hog: ${{ steps.filter.outputs.hog }}
steps:
# For pull requests it's not necessary to checkout the code, but we
# also want this to run on master so we need to checkout
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
hog:
# Avoid running tests for irrelevant changes
- 'hogvm/**/*'
- 'posthog/hogql/**/*'
- 'bin/hog'
- 'bin/hoge'
- requirements.txt
- requirements-dev.txt
- .github/workflows/ci-hog.yml
hog-tests:
needs: changes
timeout-minutes: 30
name: Hog tests
runs-on: ubuntu-24.04
if: needs.changes.outputs.hog == 'true'
steps:
# If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined
# there are backend changes, cancel previous runs
- uses: n1hility/cancel-previous-runs@v3
if: github.actor != 'posthog-bot'
with:
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.11.9
cache: 'pip'
cache-dependency-path: '**/requirements*.txt'
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
# uv is a fast pip alternative: https://github.com/astral-sh/uv/
- run: pip install uv
- name: Install SAML (python3-saml) dependencies
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl
- name: Install Python dependencies
run: |
uv pip install --system -r requirements.txt -r requirements-dev.txt
- name: Install pnpm
uses: pnpm/action-setup@v4
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: 18
- name: Check if ANTLR definitions are up to date
run: |
cd ..
sudo apt-get install default-jre
mkdir antlr
cd antlr
curl -o antlr.jar https://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar
export PWD=`pwd`
echo '#!/bin/bash' > antlr
echo "java -jar $PWD/antlr.jar \$*" >> antlr
chmod +x antlr
export CLASSPATH=".:$PWD/antlr.jar:$CLASSPATH"
export PATH="$PWD:$PATH"
cd ../posthog
antlr | grep "Version"
npm run grammar:build && git diff --exit-code
env:
# Installing a version of ANTLR compatible with what's in Homebrew as of August 2024 (version 4.13.2),
# as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml
ANTLR_VERSION: '4.13.2'
- name: Check if STL bytecode is up to date
run: |
python -m hogvm.stl.compile
git diff --exit-code
- name: Run HogVM Python tests
run: |
pytest hogvm
- name: Run HogVM TypeScript tests
run: |
cd hogvm/typescript
pnpm install --frozen-lockfile
pnpm run test
- name: Run Hog tests
run: |
cd hogvm/typescript
pnpm run build
cd ../
./test.sh && git diff --exit-code
check-package-version:
name: Check HogVM TypeScript package version and detect an update
needs: hog-tests
if: needs.hog-tests.result == 'success' && needs.changes.outputs.hog == 'true'
runs-on: ubuntu-24.04
outputs:
committed-version: ${{ steps.check-package-version.outputs.committed-version }}
published-version: ${{ steps.check-package-version.outputs.published-version }}
is-new-version: ${{ steps.check-package-version.outputs.is-new-version }}
steps:
- name: Checkout the repository
uses: actions/checkout@v2
- name: Check package version and detect an update
id: check-package-version
uses: PostHog/check-package-version@v2
with:
path: hogvm/typescript
release-hogvm:
name: Release new HogVM TypeScript version
runs-on: ubuntu-24.04
needs: check-package-version
if: needs.changes.outputs.hog == 'true' && needs.check-package-version.outputs.is-new-version == 'true'
env:
COMMITTED_VERSION: ${{ needs.check-package-version.outputs.committed-version }}
PUBLISHED_VERSION: ${{ needs.check-package-version.outputs.published-version }}
steps:
- name: Checkout the repository
uses: actions/checkout@v4
with:
fetch-depth: 1
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.11.9
cache: 'pip'
cache-dependency-path: '**/requirements*.txt'
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- run: pip install uv
- name: Install SAML (python3-saml) dependencies
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl
- name: Install Python dependencies
run: |
uv pip install --system -r requirements.txt -r requirements-dev.txt
- name: Install pnpm
uses: pnpm/action-setup@v4
- name: Set up Node 18
uses: actions/setup-node@v4
with:
node-version: 18
registry-url: https://registry.npmjs.org
- name: Install package.json dependencies
run: cd hogvm/typescript && pnpm install
- name: Publish the package in the npm registry
run: cd hogvm/typescript && npm publish --access public
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- name: Sleep 60 seconds to allow npm to update the package
run: sleep 60
update-versions:
name: Update versions in package.json
runs-on: ubuntu-24.04
needs: release-hogvm
if: always() # This ensures the job runs regardless of the result of release-hogvm
steps:
- name: Checkout the repository
uses: actions/checkout@v4
with:
fetch-depth: 1
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- name: Install pnpm
uses: pnpm/action-setup@v4
- name: Set up Node 18
uses: actions/setup-node@v4
with:
node-version: 18
registry-url: https://registry.npmjs.org
- name: Check for version mismatches
id: check-mismatch
run: |
# Extract committed version
HOGVM_VERSION=$(jq -r '.version' hogvm/typescript/package.json)
# Compare dependencies in package.json
MAIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' package.json | tr -d '^')
PLUGIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' plugin-server/package.json | tr -d '^')
echo "HOGVM_VERSION=$HOGVM_VERSION"
echo "MAIN_VERSION=$MAIN_VERSION"
echo "PLUGIN_VERSION=$PLUGIN_VERSION"
# Set output if mismatches exist
if [[ "$HOGVM_VERSION" != "$MAIN_VERSION" || "$HOGVM_VERSION" != "$PLUGIN_VERSION" ]]; then
echo "mismatch=true" >> "$GITHUB_ENV"
else
echo "mismatch=false" >> "$GITHUB_ENV"
fi
- name: Update package.json versions
if: env.mismatch == 'true'
run: |
VERSION=$(jq ".version" hogvm/typescript/package.json -r)
retry_pnpm_install() {
local retries=0
local max_retries=20 # 10 minutes total
local delay=30
while [[ $retries -lt $max_retries ]]; do
echo "Attempting pnpm install (retry $((retries+1))/$max_retries)..."
pnpm install --no-frozen-lockfile && break
echo "Install failed. Retrying in $delay seconds..."
sleep $delay
retries=$((retries + 1))
done
if [[ $retries -eq $max_retries ]]; then
echo "pnpm install failed after $max_retries attempts."
exit 1
fi
}
# Update main package.json
mv package.json package.old.json
jq --indent 4 '.dependencies."@posthog/hogvm" = "^'$VERSION'"' package.old.json > package.json
rm package.old.json
retry_pnpm_install
# Update plugin-server/package.json
cd plugin-server
mv package.json package.old.json
jq --indent 4 '.dependencies."@posthog/hogvm" = "^'$VERSION'"' package.old.json > package.json
rm package.old.json
retry_pnpm_install
- name: Commit updated package.json files
if: env.mismatch == 'true'
uses: EndBug/add-and-commit@v9
with:
add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "hogvm/typescript/package.json"]'
message: 'Update @posthog/hogvm version in package.json'
default_author: github_actions
github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}