Merge branch 'master' into fix/wait-period-wording

.github/workflows/ci-backend.yml

@@ -91,7 +91,6 @@ jobs:
 
     backend-code-quality:
         needs: changes
-        # if: needs.changes.outputs.backend == 'true' # <- Can't add this here because it's a required check
         timeout-minutes: 30
 
         name: Python code quality checks
@@ -106,12 +105,10 @@ jobs:
                   token: ${{ secrets.GITHUB_TOKEN }}
 
             - uses: actions/checkout@v3
-              if: needs.changes.outputs.backend == 'true'
               with:
                   fetch-depth: 1
 
             - name: Set up Python
-              if: needs.changes.outputs.backend == 'true'
               uses: actions/setup-python@v5
               with:
                   python-version: 3.11.9
@@ -121,40 +118,32 @@ jobs:
 
             # uv is a fast pip alternative: https://github.com/astral-sh/uv/
             - run: pip install uv
-              if: needs.changes.outputs.backend == 'true'
 
             - name: Install SAML (python3-saml) dependencies
-              if: needs.changes.outputs.backend == 'true'
               run: |
                   sudo apt-get update
                   sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl
 
             - name: Install Python dependencies
-              if: needs.changes.outputs.backend == 'true'
               run: |
                   uv pip install --system -r requirements.txt -r requirements-dev.txt
 
             - name: Check for syntax errors, import sort, and code style violations
-              if: needs.changes.outputs.backend == 'true'
               run: |
                   ruff check .
 
             - name: Check formatting
-              if: needs.changes.outputs.backend == 'true'
               run: |
                   ruff format --check --diff .
 
             - name: Add Problem Matcher
-              if: needs.changes.outputs.backend == 'true'
               run: echo "::add-matcher::.github/mypy-problem-matcher.json"
 
             - name: Check static typing
-              if: needs.changes.outputs.backend == 'true'
               run: |
                   mypy -p posthog | mypy-baseline filter
 
             - name: Check if "schema.py" is up to date
-              if: needs.changes.outputs.backend == 'true'
               run: |
                   npm run schema:build:python && git diff --exit-code
 
@@ -235,9 +224,8 @@ jobs:
 
     django:
         needs: changes
-        # if: needs.changes.outputs.backend == 'true' # <- Can't add this here because it's a required check
-
-        timeout-minutes: 30 # increase for tmate testing
+        # increase for tmate testing
+        timeout-minutes: 30
 
         name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
         runs-on: ubuntu-24.04
@@ -273,8 +261,10 @@ jobs:
                       group: 3
 
         steps:
+            # The first step is the only one that should run if `needs.changes.outputs.backend == 'false'`.
+            # All the other ones should rely on `needs.changes.outputs.backend` directly or indirectly, so that they're
+            # effectively skipped if backend code is unchanged. See https://github.com/PostHog/posthog/pull/15174.
             - uses: actions/checkout@v3
-              if: needs.changes.outputs.backend == 'true'
               with:
                   fetch-depth: 1
                   repository: ${{ github.event.pull_request.head.repo.full_name }}
@@ -317,7 +307,7 @@ jobs:
                   fi
 
             - name: Fail CI if some snapshots have been updated but not committed
-              if: needs.changes.outputs.backend == 'true' && steps.changed-files.outputs.files_found == 'true' && steps.add-and-commit.outcome == 'success'
+              if: steps.changed-files.outputs.files_found == 'true' && steps.add-and-commit.outcome == 'success'
               run: |
                   echo "${{ steps.changed-files.outputs.diff }}"
                   exit 1
@@ -333,11 +323,11 @@ jobs:
     async-migrations:
         name: Async migrations tests -  ${{ matrix.clickhouse-server-image }}
         needs: changes
-        if: needs.changes.outputs.backend == 'true'
         strategy:
             fail-fast: false
             matrix:
                 clickhouse-server-image: ['clickhouse/clickhouse-server:24.8.7.41']
+        if: needs.changes.outputs.backend == 'true'
         runs-on: ubuntu-24.04
         steps:
             - name: 'Checkout repo'
@@ -391,7 +381,7 @@ jobs:
 
     calculate-running-time:
         name: Calculate running time
-        needs: [changes, django, async-migrations]
+        needs: [django, async-migrations]
         runs-on: ubuntu-24.04
         if: # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks
             needs.changes.outputs.backend == 'true' &&

.github/workflows/ci-e2e.yml

@@ -68,7 +68,6 @@ jobs:
     container:
         name: Build and cache container image
         runs-on: ubuntu-24.04
-        if: needs.changes.outputs.shouldTriggerCypress == 'true'
         timeout-minutes: 60
         needs: [changes]
         permissions:
@@ -79,9 +78,12 @@ jobs:
             build-id: ${{ steps.build.outputs.build-id }}
         steps:
             - name: Checkout
+              if: needs.changes.outputs.shouldTriggerCypress == 'true'
               uses: actions/checkout@v3
             - name: Build the Docker image with Depot
-              uses: ./.github/actions/build-n-cache-image # Build the container image in preparation for the E2E tests
+              if: needs.changes.outputs.shouldTriggerCypress == 'true'
+              # Build the container image in preparation for the E2E tests
+              uses: ./.github/actions/build-n-cache-image
               id: build
               with:
                   save: true
@@ -166,8 +168,7 @@ jobs:
                   build-id: ${{ needs.container.outputs.build-id }}
                   tags: ${{ needs.container.outputs.tag }}
 
-            - name: Write .env
-              if: needs.changes.outputs.shouldTriggerCypress == 'true'
+            - name: Write .env # This step intentionally has no if, so that GH always considers the action as having run
               run: |
                   cat <<EOT >> .env
                   SECRET_KEY=6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da
@@ -279,7 +280,7 @@ jobs:
     calculate-running-time:
         name: Calculate running time
         runs-on: ubuntu-24.04
-        needs: [changes, cypress]
+        needs: [cypress]
         if: needs.changes.outputs.shouldTriggerCypress == 'true' &&
             github.event.pull_request.head.repo.full_name == 'PostHog/posthog'
         steps:

.github/workflows/ci-frontend.yml

@@ -52,13 +52,12 @@ jobs:
 
     frontend-code-quality:
         name: Code quality checks
-        # if: needs.changes.outputs.frontend == 'true' # <- Can't add this here because it's a required check
         needs: changes
         # kea typegen and typescript:check need some more oomph
         runs-on: ubuntu-24.04
         steps:
+            # we need at least one thing to run to make sure we include everything for required jobs
             - uses: actions/checkout@v3
-              if: needs.changes.outputs.frontend == 'true'
 
             - name: Install pnpm
               if: needs.changes.outputs.frontend == 'true'
@@ -124,7 +123,6 @@ jobs:
     jest:
         runs-on: ubuntu-24.04
         needs: changes
-        # if: needs.changes.outputs.frontend == 'true' # <- Can't add this here because it's a required check
         name: Jest test (${{ matrix.segment }} - ${{ matrix.chunk }})
 
         strategy:
@@ -135,8 +133,8 @@ jobs:
                 chunk: [1, 2, 3]
 
         steps:
+            # we need at least one thing to run to make sure we include everything for required jobs
             - uses: actions/checkout@v3
-              if: needs.changes.outputs.frontend == 'true'
 
             - name: Remove ee
               if: needs.changes.outputs.frontend == 'true' && matrix.segment == 'FOSS'

.github/workflows/ci-hog.yml

@@ -11,12 +11,10 @@ on:
         paths-ignore:
             - rust/**
             - livestream/**
-            - .github/**
     pull_request:
         paths-ignore:
             - rust/**
             - livestream/**
-            - .github/**
 
 jobs:
     # Job to decide if we should run backend ci
@@ -49,11 +47,10 @@ jobs:
 
     hog-tests:
         needs: changes
-        if: needs.changes.outputs.hog == 'true'
         timeout-minutes: 30
-
         name: Hog tests
         runs-on: ubuntu-24.04
+        if: needs.changes.outputs.hog == 'true'
 
         steps:
             # If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined
@@ -93,7 +90,7 @@ jobs:
             - name: Set up Node.js
               uses: actions/setup-node@v4
               with:
-                  node-version: 18.12.1
+                  node-version: 18
 
             - name: Check if ANTLR definitions are up to date
               run: |
@@ -138,3 +135,155 @@ jobs:
                   pnpm run build 
                   cd ../
                   ./test.sh && git diff --exit-code
+
+    check-package-version:
+        name: Check HogVM TypeScript package version and detect an update
+        needs: hog-tests
+        if: needs.hog-tests.result == 'success' && needs.changes.outputs.hog == 'true'
+        runs-on: ubuntu-24.04
+        outputs:
+            committed-version: ${{ steps.check-package-version.outputs.committed-version }}
+            published-version: ${{ steps.check-package-version.outputs.published-version }}
+            is-new-version: ${{ steps.check-package-version.outputs.is-new-version }}
+        steps:
+            - name: Checkout the repository
+              uses: actions/checkout@v2
+            - name: Check package version and detect an update
+              id: check-package-version
+              uses: PostHog/check-package-version@v2
+              with:
+                  path: hogvm/typescript
+
+    release-hogvm:
+        name: Release new HogVM TypeScript version
+        runs-on: ubuntu-24.04
+        needs: check-package-version
+        if: needs.changes.outputs.hog == 'true' && needs.check-package-version.outputs.is-new-version == 'true'
+        env:
+            COMMITTED_VERSION: ${{ needs.check-package-version.outputs.committed-version }}
+            PUBLISHED_VERSION: ${{ needs.check-package-version.outputs.published-version }}
+        steps:
+            - name: Checkout the repository
+              uses: actions/checkout@v4
+              with:
+                  fetch-depth: 1
+                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
+            - name: Set up Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: 3.11.9
+                  cache: 'pip'
+                  cache-dependency-path: '**/requirements*.txt'
+                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
+            - run: pip install uv
+            - name: Install SAML (python3-saml) dependencies
+              run: |
+                  sudo apt-get update
+                  sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl
+            - name: Install Python dependencies
+              run: |
+                  uv pip install --system -r requirements.txt -r requirements-dev.txt
+            - name: Install pnpm
+              uses: pnpm/action-setup@v4
+            - name: Set up Node 18
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 18
+                  registry-url: https://registry.npmjs.org
+            - name: Install package.json dependencies
+              run: cd hogvm/typescript && pnpm install
+            - name: Publish the package in the npm registry
+              run: cd hogvm/typescript && npm publish --access public
+              env:
+                  NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+            - name: Sleep 60 seconds to allow npm to update the package
+              run: sleep 60
+
+    update-versions:
+        name: Update versions in package.json
+        runs-on: ubuntu-24.04
+        needs: release-hogvm
+        if: always() # This ensures the job runs regardless of the result of release-hogvm
+        steps:
+            - name: Checkout the repository
+              uses: actions/checkout@v4
+              with:
+                  fetch-depth: 1
+                  repository: ${{ github.event.pull_request.head.repo.full_name }}
+                  ref: ${{ github.event.pull_request.head.ref }}
+                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
+
+            - name: Install pnpm
+              uses: pnpm/action-setup@v4
+            - name: Set up Node 18
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 18
+                  registry-url: https://registry.npmjs.org
+
+            - name: Check for version mismatches
+              id: check-mismatch
+              run: |
+                  # Extract committed version
+                  HOGVM_VERSION=$(jq -r '.version' hogvm/typescript/package.json)
+
+                  # Compare dependencies in package.json
+                  MAIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' package.json | tr -d '^')
+                  PLUGIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' plugin-server/package.json | tr -d '^')
+
+                  echo "HOGVM_VERSION=$HOGVM_VERSION"
+                  echo "MAIN_VERSION=$MAIN_VERSION"
+                  echo "PLUGIN_VERSION=$PLUGIN_VERSION"
+
+                  # Set output if mismatches exist
+                  if [[ "$HOGVM_VERSION" != "$MAIN_VERSION" || "$HOGVM_VERSION" != "$PLUGIN_VERSION" ]]; then
+                      echo "mismatch=true" >> "$GITHUB_ENV"
+                  else
+                      echo "mismatch=false" >> "$GITHUB_ENV"
+                  fi
+
+            - name: Update package.json versions
+              if: env.mismatch == 'true'
+              run: |
+                  VERSION=$(jq ".version" hogvm/typescript/package.json -r)
+
+                  retry_pnpm_install() {
+                      local retries=0
+                      local max_retries=20 # 10 minutes total
+                      local delay=30
+
+                      while [[ $retries -lt $max_retries ]]; do
+                          echo "Attempting pnpm install (retry $((retries+1))/$max_retries)..."
+                          pnpm install --no-frozen-lockfile && break
+                          echo "Install failed. Retrying in $delay seconds..."
+                          sleep $delay
+                          retries=$((retries + 1))
+                      done
+
+                      if [[ $retries -eq $max_retries ]]; then
+                          echo "pnpm install failed after $max_retries attempts."
+                          exit 1
+                      fi
+                  }
+
+                  # Update main package.json
+                  mv package.json package.old.json
+                  jq --indent 4 '.dependencies."@posthog/hogvm" = "^'$VERSION'"' package.old.json > package.json
+                  rm package.old.json
+                  retry_pnpm_install
+
+                  # Update plugin-server/package.json
+                  cd plugin-server
+                  mv package.json package.old.json
+                  jq --indent 4 '.dependencies."@posthog/hogvm" = "^'$VERSION'"' package.old.json > package.json
+                  rm package.old.json
+                  retry_pnpm_install
+
+            - name: Commit updated package.json files
+              if: env.mismatch == 'true'
+              uses: EndBug/add-and-commit@v9
+              with:
+                  add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "hogvm/typescript/package.json"]'
+                  message: 'Update @posthog/hogvm version in package.json'
+                  default_author: github_actions
+                  github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}