test: Regular checkup of visual regression tests (#18469)

* test: Regular checkup of visual regression tests

* Fix billing gauge animation

Animations done in JS can't be stopped automatically by the Storybook test runner. CSS animations can, easily, and they are anyway the cleaner and more performant way of achieving the same here.

* Rename misleading feature flag `recentInsights` to `relatedInsights`

* Mock homepage endpoints to avoid error toasts

* Wait for the recordings list in the notebook node story

* Fix `featureFlagLogic`

* Wait for `.NotebookNode__content`

* Try to optimize

* Screenshot failures and upload as artifacts

* Fix remaining failures

* Increase timeouts

* Fix rendering of Survey stories

* Remove `clang-format`

* Update UI snapshots for `chromium` (2)

* Update UI snapshots for `chromium` (2)

* Update UI snapshots for `chromium` (2)

* Fix alignment of series name in insights details

* Try to fix experiment story flakiness

* Include toasts in loaders

* Fix superfluous toast

* Fix un-awaited breakpoints

* Update UI snapshots for `chromium` (1)

* Update UI snapshots for `chromium` (1)

* Update UI snapshots for `chromium` (1)

* Make login snapshots slightly stabler

* Update UI snapshots for `chromium` (2)

* Update UI snapshots for `chromium` (2)

* Skip incorrect Surveys story

* Update UI snapshots for `chromium` (2)

* Revert msw upgrade

---------

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

.dockerignore

@@ -36,3 +36,5 @@
 !unit.json
 !plugin-transpiler/src
 !plugin-transpiler/*.*
+!test-runner-jest.config.js
+!test-runner-jest-environment.js

.github/workflows/storybook-chromatic.yml

@@ -64,7 +64,6 @@ jobs:
             SHARD_COUNT: '2'
             CYPRESS_INSTALL_BINARY: '0'
             NODE_OPTIONS: --max-old-space-size=6144
-            JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE: '1' # Remove obsolete snapshots
             OPT_OUT_CAPTURE: 1
         outputs:
             # The below have to be manually listed unfortunately, as GitHub Actions doesn't allow matrix-dependent outputs
@@ -132,7 +131,7 @@ jobs:
                   retries=3
                   while [ $retries -gt 0 ]; do
                       pnpm exec http-server storybook-static --port 6006 --silent &
-                      if pnpm wait-on http://127.0.0.1:6006 --timeout 60; then
+                      if pnpm wait-on http://127.0.0.1:6006 --timeout 15; then
                           break
                       fi
                       retries=$((retries-1))
@@ -146,14 +145,7 @@ jobs:
                   # Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
                   VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
               run: |
-                  retries=3
-                  while [ $retries -gt 0 ]; do
-                      if pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT; then
-                          break
-                      fi
-                      retries=$((retries-1))
-                      echo "Failed @storybook/test-runner, retrying... ($retries retries left)"
-                  done
+                  pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT
 
             - name: Run @playwright/test (legacy, Chromium-only)
               if: matrix.browser == 'chromium' && matrix.shard == 1
@@ -163,6 +155,13 @@ jobs:
               run: |
                   pnpm test:visual-regression:legacy:ci:$VARIANT
 
+            - name: Archive failure screenshots
+              if: ${{ failure() }}
+              uses: actions/upload-artifact@v3
+              with:
+                  name: failure-screenshots-${{ matrix.browser }}
+                  path: frontend/__snapshots__/__failures__/
+
             - name: Count and optimize updated snapshots
               id: diff
               # Skip on forks

.gitignore

@@ -18,6 +18,7 @@ frontend/.cache/
 frontend/dist/
 frontend/types/
 frontend/__snapshots__/__diff_output__/
+frontend/__snapshots__/__failures__/
 *Type.ts
 frontend/pnpm-error.log
 frontend/tmp

.storybook/test-runner.ts

@@ -48,36 +48,35 @@ declare module '@storybook/types' {
     }
 }
 
-const RETRY_TIMES = 5
+const RETRY_TIMES = 3
 const LOADER_SELECTORS = [
     '.ant-skeleton',
     '.Spinner',
     '.LemonSkeleton',
     '.LemonTableLoader',
+    '.Toastify__toast-container',
     '[aria-busy="true"]',
-    '[aria-label="Content is loading..."]',
     '.SessionRecordingPlayer--buffering',
     '.Lettermark--unknown',
 ]
 
 const customSnapshotsDir = `${process.cwd()}/frontend/__snapshots__`
 
-const TEST_TIMEOUT_MS = 10000
-const BROWSER_DEFAULT_TIMEOUT_MS = 9000 // Reduce the default timeout down from 30s, to pre-empt Jest timeouts
-const SCREENSHOT_TIMEOUT_MS = 9000
+const JEST_TIMEOUT_MS = 15000
+const PLAYWRIGHT_TIMEOUT_MS = 10000 // Must be shorter than JEST_TIMEOUT_MS
 
 module.exports = {
     setup() {
         expect.extend({ toMatchImageSnapshot })
         jest.retryTimes(RETRY_TIMES, { logErrorsBeforeRetry: true })
-        jest.setTimeout(TEST_TIMEOUT_MS)
+        jest.setTimeout(JEST_TIMEOUT_MS)
     },
     async postRender(page, context) {
         const browserContext = page.context()
         const storyContext = (await getStoryContext(page, context)) as StoryContext
         const { skip = false, snapshotBrowsers = ['chromium'] } = storyContext.parameters?.testOptions ?? {}
 
-        browserContext.setDefaultTimeout(BROWSER_DEFAULT_TIMEOUT_MS)
+        browserContext.setDefaultTimeout(PLAYWRIGHT_TIMEOUT_MS)
         if (!skip) {
             const currentBrowser = browserContext.browser()!.browserType().name() as SupportedBrowserName
             if (snapshotBrowsers.includes(currentBrowser)) {
@@ -202,7 +201,7 @@ async function expectLocatorToMatchStorySnapshot(
     browser: SupportedBrowserName,
     options?: LocatorScreenshotOptions
 ): Promise<void> {
-    const image = await locator.screenshot({ timeout: SCREENSHOT_TIMEOUT_MS, ...options })
+    const image = await locator.screenshot({ ...options })
     let customSnapshotIdentifier = context.id
     if (browser !== 'chromium') {
         customSnapshotIdentifier += `--${browser}`

Dockerfile.playwright

@@ -15,6 +15,6 @@ ENV CYPRESS_INSTALL_BINARY=0
 
 RUN pnpm install --frozen-lockfile
 
-COPY playwright.config.ts webpack.config.js babel.config.js tsconfig.json ./
+COPY playwright.config.ts webpack.config.js babel.config.js tsconfig.json test-runner-jest.config.js test-runner-jest-environment.js ./
 
 COPY .storybook/ .storybook/

frontend/src/lib/components/ActivationSidebar/activationLogic.ts

@@ -137,7 +137,7 @@ export const activationLogic = kea<activationLogicType>([
             0,
             {
                 loadCustomEvents: async (_, breakpoint) => {
-                    breakpoint(200)
+                    await breakpoint(200)
                     const url = api.eventDefinitions.determineListEndpoint({
                         event_type: EventDefinitionType.EventCustom,
                     })

frontend/src/lib/components/Cards/InsightCard/InsightCard.scss

@@ -127,6 +127,7 @@
 
 .SeriesDisplay__raw-name {
     display: inline-flex;
+    align-items: center;
     padding: 0.125rem 0.25rem;
     margin: 0 0.25rem;
     background: var(--primary-bg-hover);
@@ -135,12 +136,12 @@
     font-size: 0.6875rem;
     font-weight: 600;
     line-height: 1rem;
-    vertical-align: -0.3em;
     &.SeriesDisplay__raw-name--action,
     &.SeriesDisplay__raw-name--event {
         padding: 0.25rem;
         &::before {
             display: inline-block;
+            flex-shrink: 0;
             text-align: center;
             width: 1rem;
             border-radius: 0.25rem;
@@ -169,5 +170,4 @@
     margin-right: 0.25rem;
     color: var(--border-bold);
     font-size: 1.25rem;
-    vertical-align: middle;
 }

frontend/src/lib/components/Map/Map.stories.tsx

@@ -14,6 +14,11 @@ const meta: Meta<typeof Map> = {
         center: coordinates,
         className: 'h-60',
     },
+    parameters: {
+        testOptions: {
+            skip: true,
+        },
+    },
 }
 type Story = StoryObj<typeof Map>
 

frontend/src/lib/components/PropertyFilters/components/TaxonomicPropertyFilter.tsx

@@ -109,7 +109,7 @@ export function TaxonomicPropertyFilter({
 
     const { ref: wrapperRef, size } = useResizeBreakpoints({
         0: 'tiny',
-        400: 'small',
+        300: 'small',
         550: 'medium',
     })
 

frontend/src/lib/components/SceneDashboardChoice/sceneDashboardChoiceModalLogic.test.ts

@@ -7,7 +7,7 @@ import { userLogic } from 'scenes/userLogic'
 import { useMocks } from '~/mocks/jest'
 import { Scene } from 'scenes/sceneTypes'
 
-describe('sceneDashboardChoiceModalLogic ', () => {
+describe('sceneDashboardChoiceModalLogic', () => {
     let logic: ReturnType<typeof sceneDashboardChoiceModalLogic.build>
 
     beforeEach(async () => {

frontend/src/scenes/authentication/Login.stories.tsx

@@ -1,5 +1,5 @@
 // Login.stories.tsx
-import { Meta } from '@storybook/react'
+import { Meta, StoryFn } from '@storybook/react'
 import { Login } from './Login'
 import { mswDecorator, useStorybookMocks } from '~/mocks/browser'
 import { useEffect } from 'react'
@@ -27,7 +27,8 @@ const meta: Meta = {
     ],
 }
 export default meta
-export const Cloud = (): JSX.Element => {
+
+export const Cloud: StoryFn = () => {
     useStorybookMocks({
         get: {
             '/_preflight': {
@@ -41,7 +42,8 @@ export const Cloud = (): JSX.Element => {
     })
     return <Login />
 }
-export const CloudEU = (): JSX.Element => {
+
+export const CloudEU: StoryFn = () => {
     useStorybookMocks({
         get: {
             '/_preflight': {
@@ -56,7 +58,8 @@ export const CloudEU = (): JSX.Element => {
     })
     return <Login />
 }
-export const CloudWithGoogleLoginEnforcement = (): JSX.Element => {
+
+export const CloudWithGoogleLoginEnforcement: StoryFn = () => {
     useStorybookMocks({
         get: {
             '/_preflight': {
@@ -78,7 +81,13 @@ export const CloudWithGoogleLoginEnforcement = (): JSX.Element => {
     }, [])
     return <Login />
 }
-export const SelfHosted = (): JSX.Element => {
+CloudWithGoogleLoginEnforcement.parameters = {
+    testOptions: {
+        waitForSelector: '[href^="/login/google-oauth2/"]',
+    },
+}
+
+export const SelfHosted: StoryFn = () => {
     useStorybookMocks({
         get: {
             '/_preflight': {
@@ -92,7 +101,7 @@ export const SelfHosted = (): JSX.Element => {
     return <Login />
 }
 
-export const SelfHostedWithSAML = (): JSX.Element => {
+export const SelfHostedWithSAML: StoryFn = () => {
     useStorybookMocks({
         get: {
             '/_preflight': {
@@ -105,8 +114,13 @@ export const SelfHostedWithSAML = (): JSX.Element => {
     })
     return <Login />
 }
+SelfHostedWithSAML.parameters = {
+    testOptions: {
+        waitForSelector: '[href^="/login/saml/"]',
+    },
+}
 
-export const SSOError = (): JSX.Element => {
+export const SSOError: StoryFn = () => {
     useStorybookMocks({
         get: {
             '/_preflight': preflightJson,
@@ -119,7 +133,7 @@ export const SSOError = (): JSX.Element => {
     return <Login />
 }
 
-export const SecondFactor = (): JSX.Element => {
+export const SecondFactor: StoryFn = () => {
     useEffect(() => {
         // Change the URL
         router.actions.push(urls.login2FA())

frontend/src/scenes/billing/BillingGauge.scss

@@ -1,8 +1,5 @@
-.BillingGauge {
-}
-
 .BillingGaugeItem {
-    transition: all 1000ms cubic-bezier(0.15, 0.15, 0.2, 1);
+    animation: billing-gauge-item-expand 800ms cubic-bezier(0.15, 0.15, 0.2, 1) forwards;
 
     .BillingGaugeItem__info {
         position: absolute;
@@ -28,3 +25,12 @@
         }
     }
 }
+
+@keyframes billing-gauge-item-expand {
+    0% {
+        width: 0%;
+    }
+    100% {
+        width: var(--billing-gauge-item-width);
+    }
+}

frontend/src/scenes/billing/BillingGauge.tsx

@@ -1,7 +1,7 @@
 import clsx from 'clsx'
 import { Tooltip } from 'lib/lemon-ui/Tooltip'
 import { compactNumber } from 'lib/utils'
-import { useEffect, useMemo, useState } from 'react'
+import { useMemo } from 'react'
 import './BillingGauge.scss'
 
 type BillingGaugeItemProps = {
@@ -15,7 +15,10 @@ type BillingGaugeItemProps = {
 const BillingGaugeItem = ({ width, className, tooltip, top, value }: BillingGaugeItemProps): JSX.Element => {
     return (
         // eslint-disable-next-line react/forbid-dom-props
-        <div className={`BillingGaugeItem absolute top-0 left-0 bottom-0 h-2 ${className}`} style={{ width: width }}>
+        <div
+            className={`BillingGaugeItem absolute top-0 left-0 bottom-0 h-2 ${className}`}
+            style={{ '--billing-gauge-item-width': width } as React.CSSProperties}
+        >
             <div className="absolute right-0 w-px h-full bg-bg-light" />
             <Tooltip title={value.toLocaleString()} placement={'right'}>
                 <div
@@ -41,22 +44,16 @@ export type BillingGaugeProps = {
 }
 
 export function BillingGauge({ items }: BillingGaugeProps): JSX.Element {
-    const [expanded, setExpanded] = useState(false)
     const maxScale = useMemo(() => {
         return Math.max(100, ...items.map((item) => item.value)) * 1.3
     }, [items])
 
-    useEffect(() => {
-        // On mount, animate the gauge to full width
-        setExpanded(true)
-    }, [])
-
     return (
         <div className="relative h-2 bg-border-light my-16">
             {items.map((item, i) => (
                 <BillingGaugeItem
                     key={i}
-                    width={expanded ? `${(item.value / maxScale) * 100}%` : '0%'}
+                    width={`${(item.value / maxScale) * 100}%`}
                     className={`bg-${item.color}`}
                     tooltip={<b>{item.text}</b>}
                     top={item.top}

frontend/src/scenes/billing/billingLogic.ts

@@ -231,7 +231,7 @@ export const billingLogic = kea<billingLogicType>([
                 license: !license ? 'Please enter your license key' : undefined,
             }),
             submit: async ({ license }, breakpoint) => {
-                breakpoint(500)
+                await breakpoint(500)
                 try {
                     await api.update('api/billing-v2/license', {
                         license,