Skip to content

Commit

Permalink
test(frontend): Automatically update UI snapshots in CI (#14391)
Browse files Browse the repository at this point in the history
* Update UI snapshots with comment instead of failing on change

* Make a few stories more reliable

* Remove unused snapshots in CI

* Update query snapshots

* Update `webkit` UI snapshots

* Update `chromium` UI snapshots

* Update `firefox` UI snapshots

* Shard visual regression tests

* Update UI snapshots for `chromium` (1)

* Update UI snapshots for `webkit` (2)

* Fix remaining issues

* Update UI snapshots for `chromium` (1)

* Update UI snapshots for `chromium` (2)

* Update UI snapshots for `chromium` (2)

* Update UI snapshots for `chromium` (1)

* Update UI snapshots for `chromium` (1)

* Improve reliability of insights snapshots

* Update UI snapshots for `chromium` (1)

* Update UI snapshots for `chromium` (2)

* Improve reliability of login/signup snapshots

* Update UI snapshots for `chromium` (2)

* Update UI snapshots for `chromium` (1)

* Update .github/workflows/storybook-chromatic.yml

Co-authored-by: Thomas Obermüller <[email protected]>

---------

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Thomas Obermüller <[email protected]>
  • Loading branch information
3 people authored Feb 24, 2023
1 parent 192af82 commit 1c2d74f
Showing 126 changed files with 344 additions and 123 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/ci-backend.yml
Original file line number Diff line number Diff line change
@@ -268,8 +268,9 @@ jobs:
if: ${{ !matrix.person-on-events && github.event.pull_request.head.repo.full_name == github.repository }}
with:
add: '["ee", "posthog/clickhouse/test/__snapshots__", "posthog/api/test/__snapshots__", "posthog/test/__snapshots__", "posthog/queries/"]'
message: 'Update snapshots'
message: 'Update query snapshots'
default_author: github_actions
github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

- name: Check if any snapshot changes were left uncomitted
id: changed-files
61 changes: 0 additions & 61 deletions .github/workflows/ci-frontend.yml
Original file line number Diff line number Diff line change
@@ -113,64 +113,3 @@ jobs:
env:
NODE_OPTIONS: --max-old-space-size=6144
CHUNKS: ${{ needs.jest-setup.outputs['test-chunks'] }}

visual-regression:
name: Visual regression tests
runs-on: ubuntu-20.04
container:
image: mcr.microsoft.com/playwright:v1.29.2-focal
strategy:
fail-fast: false
matrix:
browser: ['chromium', 'firefox', 'webkit']
env:
CYPRESS_INSTALL_BINARY: '0'
NODE_OPTIONS: --max_old_space_size=4096
steps:
- uses: actions/checkout@v3

- name: Install pnpm
uses: pnpm/action-setup@v2
with:
version: 7.x.x

- name: Set up Node.js
uses: actions/setup-node@v3
with:
node-version: 18
cache: pnpm

- name: Install package.json dependencies with pnpm
run: pnpm install --frozen-lockfile

- name: Install CI utilities with pnpm
run: pnpm install http-server wait-on

- name: Build Storybook
run: pnpm build-storybook --quiet # Silence since progress logging results in a massive wall of spam

- name: Serve Storybook in the background
run: pnpm exec http-server storybook-static --port 6006 --silent &

- name: Run @storybook/test-runner
env:
HOME: /root # Solves https://github.com/microsoft/playwright/issues/6500
run: |
pnpm wait-on http://127.0.0.1:6006 --timeout 60 # Wait for the server to be ready
pnpm test:visual-regression:stories:ci ${{ matrix.browser }}
- name: Run @playwright/test (legacy, Chromium-only)
if: ${{ matrix.browser == 'chromium' }}
run: |
pnpm test:visual-regression:legacy:ci
- name: Upload Playwright report and diffs
uses: actions/upload-artifact@v3
if: failure()
with:
name: visual-regression-report-${{ matrix.browser }}
path: |
playwright-report/
test-results/
frontend/__snapshots__/__diff_output__/
retention-days: 7
237 changes: 222 additions & 15 deletions .github/workflows/storybook-chromatic.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,23 @@
name: 'Storybook Chromatic'
name: Storybook

on: pull_request
on:
pull_request:
paths: # Only run if the frontend has changed
- 'frontend/**'
- '.storybook/**'

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
# Allow snapshot updates to finish, so that we don't need to wait for Storybook to build again
cancel-in-progress: false

jobs:
storybook-chromatic:
name: Publish to Chromatic
runs-on: ubuntu-latest
if: github.event.pull_request.head.repo.full_name == github.repository # Don't run on forks
outputs:
storybook-url: ${{ steps.publish.outputs.storybookUrl }}
steps:
- uses: actions/checkout@v3
with:
@@ -22,24 +34,219 @@ jobs:
node-version: 18
cache: pnpm

# there's no need to run chromatic on every commit,
# so we only run it if the frontend has changed
- uses: dorny/paths-filter@v2
id: changes
with:
filters: |
frontend:
- 'frontend/**'
- '.storybook/**'
- name: Install dependencies and chromatic
if: steps.changes.outputs.frontend == 'true'
- name: Install dependencies and Chromatic
run: pnpm i -D chromatic

- name: Publish to Chromatic
if: steps.changes.outputs.frontend == 'true'
uses: chromaui/action@v1
id: publish
with:
token: ${{ secrets.GITHUB_TOKEN }}
# 👇 Chromatic projectToken, refer to the manage page to obtain it.
projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}

visual-regression:
name: Visual regression tests
runs-on: ubuntu-20.04
container:
image: mcr.microsoft.com/playwright:v1.29.2-focal
strategy:
fail-fast: false
matrix:
browser: ['chromium', 'webkit', 'firefox']
shard: [1, 2]
env:
SHARD_COUNT: '2'
CYPRESS_INSTALL_BINARY: '0'
NODE_OPTIONS: --max-old-space-size=4096
JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE: '1' # Remove obsolete snapshots
outputs:
# The below have to be manually listed unfortunately, as GitHub Actions doesn't allow matrix-dependent outputs
chromium-1-added: ${{ steps.diff.outputs.chromium-1-added }}
chromium-1-modified: ${{ steps.diff.outputs.chromium-1-modified }}
chromium-1-deleted: ${{ steps.diff.outputs.chromium-1-deleted }}
chromium-1-total: ${{ steps.diff.outputs.chromium-1-total }}
chromium-1-commitHash: ${{ steps.commit-hash.outputs.chromium-1-commitHash }}
chromium-2-added: ${{ steps.diff.outputs.chromium-2-added }}
chromium-2-modified: ${{ steps.diff.outputs.chromium-2-modified }}
chromium-2-deleted: ${{ steps.diff.outputs.chromium-2-deleted }}
chromium-2-total: ${{ steps.diff.outputs.chromium-2-total }}
chromium-2-commitHash: ${{ steps.commit-hash.outputs.chromium-2-commitHash }}
webkit-1-added: ${{ steps.diff.outputs.webkit-1-added }}
webkit-1-modified: ${{ steps.diff.outputs.webkit-1-modified }}
webkit-1-deleted: ${{ steps.diff.outputs.webkit-1-deleted }}
webkit-1-total: ${{ steps.diff.outputs.webkit-1-total }}
webkit-1-commitHash: ${{ steps.commit-hash.outputs.webkit-1-commitHash }}
webkit-2-added: ${{ steps.diff.outputs.webkit-2-added }}
webkit-2-modified: ${{ steps.diff.outputs.webkit-2-modified }}
webkit-2-deleted: ${{ steps.diff.outputs.webkit-2-deleted }}
webkit-2-total: ${{ steps.diff.outputs.webkit-2-total }}
webkit-2-commitHash: ${{ steps.commit-hash.outputs.webkit-2-commitHash }}
firefox-1-added: ${{ steps.diff.outputs.firefox-1-added }}
firefox-1-modified: ${{ steps.diff.outputs.firefox-1-modified }}
firefox-1-deleted: ${{ steps.diff.outputs.firefox-1-deleted }}
firefox-1-total: ${{ steps.diff.outputs.firefox-1-total }}
firefox-1-commitHash: ${{ steps.commit-hash.outputs.firefox-1-commitHash }}
firefox-2-added: ${{ steps.diff.outputs.firefox-2-added }}
firefox-2-modified: ${{ steps.diff.outputs.firefox-2-modified }}
firefox-2-deleted: ${{ steps.diff.outputs.firefox-2-deleted }}
firefox-2-total: ${{ steps.diff.outputs.firefox-2-total }}
firefox-2-commitHash: ${{ steps.commit-hash.outputs.firefox-2-commitHash }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
# Use PostHog Bot token when not on forks to enable proper snapshot updating
token: ${{ github.event.pull_request.head.repo.full_name == github.repository && secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}

- name: Install pnpm
uses: pnpm/action-setup@v2
with:
version: 7.x.x

- name: Set up Node.js
uses: actions/setup-node@v3
with:
node-version: 18
cache: pnpm

- name: Install package.json dependencies with pnpm
run: pnpm install --frozen-lockfile

- name: Install CI utilities with pnpm
run: pnpm install http-server wait-on

- name: Build Storybook
run: pnpm build-storybook --quiet # Silence since progress logging results in a massive wall of spam

- name: Serve Storybook in the background
run: |
pnpm exec http-server storybook-static --port 6006 --silent &
pnpm wait-on http://127.0.0.1:6006 --timeout 60 # Wait for the server to be ready
- name: Run @storybook/test-runner
env:
# Solving this bug by overriding $HOME: https://github.com/microsoft/playwright/issues/6500
HOME: /root
# Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
run: |
pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT
- name: Run @playwright/test (legacy, Chromium-only)
if: matrix.browser == 'chromium' && matrix.shard == 1
env:
# Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
run: |
pnpm test:visual-regression:legacy:ci:$VARIANT
- name: Count snapshot changes from git diff
id: diff
# Skip on forks
if: github.event.pull_request.head.repo.full_name == github.repository
run: |
git config --global --add safe.directory '*' # Calm git down about file ownership
ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l)
MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l)
DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l)
TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l)
echo A $ADDED + M $MODIFIED + D $DELETED = $TOTAL # Debugging
echo "${{ matrix.browser }}-${{ matrix.shard }}-added=$ADDED" >> $GITHUB_OUTPUT
echo "${{ matrix.browser }}-${{ matrix.shard }}-modified=$MODIFIED" >> $GITHUB_OUTPUT
echo "${{ matrix.browser }}-${{ matrix.shard }}-deleted=$DELETED" >> $GITHUB_OUTPUT
echo "${{ matrix.browser }}-${{ matrix.shard }}-total=$TOTAL" >> $GITHUB_OUTPUT
- name: Commit updated snapshots
uses: EndBug/add-and-commit@v9
# Skip on forks
if: github.event.pull_request.head.repo.full_name == github.repository
id: commit
with:
add: '["frontend/__snapshots__/", "playwright/"]'
message: 'Update UI snapshots for `${{ matrix.browser }}` (${{ matrix.shard }})'
pull: --rebase --autostash # Make sure we're up to date with other browsers' updates
default_author: github_actions
github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

- name: Add commit hash to outputs, including browser name
id: commit-hash
if: steps.commit.outputs.pushed == 'true'
run: echo "${{ matrix.browser }}-${{ matrix.shard }}-commitHash=${{ steps.commit.outputs.commit_long_sha }}" >> $GITHUB_OUTPUT

visual-regression-summary:
name: Summarize visual regression tests
runs-on: ubuntu-20.04
needs: visual-regression
if: always() # Run even if visual-regression fails for one (or more) of the browsers
steps:
- name: Post comment about updated snapshots
if: github.event.pull_request.head.repo.full_name == github.repository
uses: actions/github-script@v6
with:
github-token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
script: |
const BROWSERS = ['chromium', 'webkit', 'firefox']
const diffJobOutputs = ${{ toJson(needs.visual-regression.outputs) }}
const summaryDiff = { total: 0, added: 0, modified: 0, deleted: 0 }
const diffByBrowser = Object.fromEntries(BROWSERS.map(browser => [browser, {
total: 0, added: 0, modified: 0, deleted: 0, commitHashes: []
}]))
for (const [key, rawValue] of Object.entries(diffJobOutputs)) {
// Split e.g. 'chromium-1-commitHash' into ['chromium', '1' 'commitHash']
const [browser, shardNumber, diffKey] = key.split('-')
// Sum up the counts - but not the commit hash
if (diffKey === 'commitHash') {
diffByBrowser[browser].commitHashes.push([parseInt(shardNumber), rawValue])
} else {
const value = parseInt(rawValue)
diffByBrowser[browser][diffKey] += value
summaryDiff[diffKey] += value
}
}
for (const browser of BROWSERS) {
if (diffByBrowser[browser]?.total === undefined) {
diffByBrowser[browser] = null // Null means failure
}
}
if (summaryDiff.total === 0) {
console.log('No changes were made, skipping comment')
return
}
const diffByBrowserDisplay = Object.entries(diffByBrowser).map(([browser, diff]) => {
if (!diff) {
return `- \`${browser}\`: failed`
}
const { added: a, modified: m, deleted: d, commitHashes } = diff
const b = a + m + d > 0 ? '**' : '' // Bold list item if there were changes
let extraInfo = ''
if (b) {
const commitInfo = commitHashes.map(
([shardNumber, commitHash]) =>
`[diff for shard ${shardNumber}](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${commitHash})`
).join(', ') || "wasn't pushed!"
extraInfo = ` (${commitInfo})`
}
return `- ${b}\`${browser}\`${b}: **${a}** added, **${m}** modified, **${d}** deleted${extraInfo}`
}).join('\n')
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: `## 📸 UI snapshots have been updated
**${summaryDiff.total}** snapshot changes in total. **${summaryDiff.added}** added, **${summaryDiff.modified}** modified, **${summaryDiff.deleted}** deleted:
${diffByBrowserDisplay}
Triggered by [this commit](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${{ github.sha }}).
👉 **[Review this PR's diff of snapshots.](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/files#:~:text=frontend/__snapshots__/)**`
})
Loading

0 comments on commit 1c2d74f

Please sign in to comment.