chore: Run snapshots in separate tests, once

.github/actions/run-backend-tests/action.yml

@@ -25,6 +25,9 @@ inputs:
     token:
         required: false
         description: GitHub token
+    snapshot_tests:
+        required: true
+        description: Whether to only run the snapshot tests or not
 
 runs:
     using: 'composite'
@@ -108,7 +111,7 @@ runs:
                   inputs.person-on-events == 'true'
                   && './posthog/clickhouse/ ./posthog/hogql/ ./posthog/queries/ ./posthog/api/test/test_insight* ./posthog/api/test/dashboards/test_dashboard.py'
                   || 'hogvm posthog'
-              }} -m "not async_migrations" \
+              }} -m "not async_migrations and ${{inputs.snapshot_tests && 'snapshot_tests' || 'not snapshot_tests'}}" \
                   --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \
                   --durations=100 --durations-min=1.0 \
                   $PYTEST_ARGS
@@ -120,7 +123,7 @@ runs:
               GROUPS_ON_EVENTS_ENABLED: ${{ inputs.person-on-events }}
           shell: bash
           run: | # async_migrations covered in ci-async-migrations.yml
-              pytest ${{ inputs.person-on-events == 'true' && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations" \
+              pytest ${{ inputs.person-on-events == 'true' && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations ${{inputs.snapshot_tests && 'snapshot_tests' || 'not snapshot_tests'}}" \
                   --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \
                   --durations=100 --durations-min=1.0 \
                   $PYTEST_ARGS

.github/workflows/ci-backend.yml

@@ -271,7 +271,36 @@ jobs:
                   concurrency: ${{ matrix.concurrency }}
                   group: ${{ matrix.group }}
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
+                  snapshot_tests: false
 
+            - name: Archive email renders
+              uses: actions/upload-artifact@v3
+              if: needs.changes.outputs.backend == 'true' && matrix.segment == 'FOSS' && matrix.person-on-events == false
+              with:
+                  name: email_renders
+                  path: posthog/tasks/test/__emails__
+                  retention-days: 5
+
+    snapshots:
+        needs: changes
+        timeout-minutes: 30
+
+        name: Snapshot tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
+        runs-on: ubuntu-latest
+
+        strategy:
+            fail-fast: false
+            matrix:
+                python-version: ['3.10.10']
+                clickhouse-server-image: ['clickhouse/clickhouse-server:23.6.1.1524']
+                segment: ['FOSS', 'EE']
+                person-on-events: [false, true]
+                # :NOTE: Keep concurrency and groups in sync
+                concurrency: [1]
+                group: [1]
+                snapshot_tests: true
+
+        steps:
             - uses: EndBug/add-and-commit@v9
               # Skip on forks
               # Also skip for persons-on-events runs, as we want to ignore snapshots diverging there
@@ -301,14 +330,6 @@ jobs:
                   echo "${{ steps.changed-files.outputs.diff }}"
                   exit 1
 
-            - name: Archive email renders
-              uses: actions/upload-artifact@v3
-              if: needs.changes.outputs.backend == 'true' && matrix.segment == 'FOSS' && matrix.person-on-events == false
-              with:
-                  name: email_renders
-                  path: posthog/tasks/test/__emails__
-                  retention-days: 5
-
     async-migrations:
         name: Async migrations tests
         needs: changes

ee/clickhouse/queries/test/test_paths.py

@@ -1,4 +1,5 @@
 from datetime import timedelta
+import pytest
 from typing import Tuple
 from unittest.mock import MagicMock
 from uuid import UUID
@@ -1731,6 +1732,7 @@ def test_event_inclusion_exclusion_filters(self):
         )
 
     @snapshot_clickhouse_queries
+    @pytest.mark.snapshot_tests
     @freeze_time("2023-05-23T11:00:00.000Z")
     def test_event_exclusion_filters_with_wildcard_groups(self):
         # P1 for pageview event /2/bar/1/foo

pytest.ini

@@ -9,4 +9,5 @@ markers =
     ee
     clickhouse_only
     skip_on_multitenancy
-    async_migrations
+    async_migrations
+    snapshot_tests