feat: Added recording of e2e test runs (#16046)

Marius nerd-sniped us to try this out. Recording of Cypress runs.

Co-authored-by: Paul D'Ambra <[email protected]>
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

.github/workflows/ci-e2e.yml

@@ -156,7 +156,7 @@ jobs:
                   KAFKA_HOSTS=kafka:9092
                   DISABLE_SECURE_SSL_REDIRECT=1
                   SECURE_COOKIES=0
-                  OPT_OUT_CAPTURE=1
+                  OPT_OUT_CAPTURE=0
                   SELF_CAPTURE=0
                   E2E_TESTING=1
                   SKIP_SERVICE_VERSION_REQUIREMENTS=1
@@ -168,6 +168,7 @@ jobs:
                   OBJECT_STORAGE_ENDPOINT=http://localhost:19000
                   OBJECT_STORAGE_ACCESS_KEY_ID=object_storage_root_user
                   OBJECT_STORAGE_SECRET_ACCESS_KEY=object_storage_root_password
+                  GITHUB_ACTION_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                   EOT
 
             - name: Lowercase GITHUB_REPOSITORY

.github/workflows/storybook-chromatic.yml

@@ -58,6 +58,7 @@ jobs:
             CYPRESS_INSTALL_BINARY: '0'
             NODE_OPTIONS: --max-old-space-size=6144
             JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE: '1' # Remove obsolete snapshots
+            OPT_OUT_CAPTURE: 1
         outputs:
             # The below have to be manually listed unfortunately, as GitHub Actions doesn't allow matrix-dependent outputs
             chromium-1-added: ${{ steps.diff.outputs.chromium-1-added }}

bin/e2e-test-runner

@@ -49,7 +49,7 @@ done
 export DEBUG=1
 export NO_RESTART_LOOP=1
 export CYPRESS_BASE_URL=http://localhost:8080
-export OPT_OUT_CAPTURE=1
+export OPT_OUT_CAPTURE="${OPT_OUT_CAPTURE:=1}"
 export SECURE_COOKIES=0
 export SKIP_SERVICE_VERSION_REQUIREMENTS=1
 export KAFKA_HOSTS=kafka:9092

cypress/support/e2e.ts

@@ -16,6 +16,10 @@ Cypress.on('window:before:load', (win) => {
 })
 
 beforeEach(() => {
+    Cypress.env('POSTHOG_PROPERTY_CURRENT_TEST_TITLE', Cypress.currentTest.title)
+    Cypress.env('POSTHOG_PROPERTY_CURRENT_TEST_FULL_TITLE', Cypress.currentTest.titlePath.join(' > '))
+    Cypress.env('POSTHOG_PROPERTY_GITHUB_ACTION_RUN_URL', process.env.GITHUB_ACTION_RUN_URL)
+
     cy.intercept('api/prompts/my_prompts/', { sequences: [], state: {} })
 
     cy.intercept('https://app.posthog.com/decide/*', (req) =>
@@ -56,6 +60,22 @@ beforeEach(() => {
     }
 })
 
+afterEach(() => {
+    if (process.env.E2E_TESTING) {
+        cy.window().then((win) => {
+            ;(win as any).posthog?.capture('e2e_testing_test_passed')
+        })
+    }
+})
+
+Cypress.on('fail', (error: Cypress.CypressError) => {
+    if (process.env.E2E_TESTING) {
+        cy.window().then((win) => {
+            ;(win as any).posthog?.capture('e2e_testing_test_failed', { e2e_testing_error_message: error.message })
+        })
+    }
+})
+
 const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/
 Cypress.on('uncaught:exception', (err) => {
     /* returning false here prevents Cypress from failing the test */

frontend/src/loadPostHogJS.tsx

@@ -40,6 +40,17 @@ export function loadPostHogJS(): void {
             })
         )
 
+        const Cypress = (window as any).Cypress
+        if (Cypress) {
+            Object.entries(Cypress.env()).forEach(([key, value]) => {
+                if (key.startsWith('POSTHOG_PROPERTY_')) {
+                    posthog.register_for_session({
+                        [key.replace('POSTHOG_PROPERTY_', 'E2E_TESTING_').toLowerCase()]: value,
+                    })
+                }
+            })
+        }
+
         // This is a helpful flag to set to automatically reset the recording session on load for testing multiple recordings
         const shouldResetSessionOnLoad = posthog.getFeatureFlag(FEATURE_FLAGS.SESSION_RESET_ON_LOAD)
         if (shouldResetSessionOnLoad) {

posthog/apps.py

@@ -22,7 +22,10 @@ def ready(self):
         posthoganalytics.personal_api_key = os.environ.get("POSTHOG_PERSONAL_API_KEY")
         posthoganalytics.poll_interval = 90
 
-        if settings.TEST or os.environ.get("OPT_OUT_CAPTURE", False):
+        if settings.E2E_TESTING:
+            posthoganalytics.api_key = "phc_ex7Mnvi4DqeB6xSQoXU1UVPzAmUIpiciRKQQXGGTYQO"
+            posthoganalytics.personal_api_key = None
+        elif settings.TEST or os.environ.get("OPT_OUT_CAPTURE", False):
             posthoganalytics.disabled = True
         elif settings.DEBUG:
             # log development server launch to posthog

posthog/settings/base_variables.py

@@ -20,6 +20,7 @@
 E2E_TESTING = get_from_env(
     "E2E_TESTING", False, type_cast=str_to_bool
 )  # whether the app is currently running for E2E tests
+OPT_OUT_CAPTURE = get_from_env("OPT_OUT_CAPTURE", False, type_cast=str_to_bool)
 BENCHMARK = get_from_env("BENCHMARK", False, type_cast=str_to_bool)
 if E2E_TESTING:
     logger.warning(

posthog/utils.py

@@ -319,7 +319,7 @@ def render_template(
 
     template = get_template(template_name)
 
-    context["opt_out_capture"] = os.getenv("OPT_OUT_CAPTURE", False)
+    context["opt_out_capture"] = settings.OPT_OUT_CAPTURE
     context["impersonated_session"] = is_impersonated_session(request)
     context["self_capture"] = settings.SELF_CAPTURE
 
@@ -335,8 +335,9 @@ def render_template(
 
     if settings.E2E_TESTING:
         context["e2e_testing"] = True
-
-    if settings.SELF_CAPTURE:
+        context["js_posthog_api_key"] = "'phc_ex7Mnvi4DqeB6xSQoXU1UVPzAmUIpiciRKQQXGGTYQO'"
+        context["js_posthog_host"] = "'https://app.posthog.com'"
+    elif settings.SELF_CAPTURE:
         api_token = get_self_capture_api_token(request)
 
         if api_token: