-
Notifications
You must be signed in to change notification settings - Fork 1.4k
297 lines (262 loc) · 15.3 KB
/
storybook-chromatic.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
name: Storybook
on:
pull_request:
paths: # Only run if the frontend has changed
- 'frontend/**'
- '.storybook/**'
- 'package.json'
- '.github/workflows/storybook-chromatic.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
# This is so that the workflow run isn't canceled when a snapshot update is pushed within it by posthog-bot
# We do however cancel from container-images-ci.yml if a commit is pushed by someone OTHER than posthog-bot
cancel-in-progress: false
jobs:
storybook-chromatic:
name: Publish to Chromatic
runs-on: ubuntu-latest
timeout-minutes: 15
if: github.event.pull_request.head.repo.full_name == github.repository # Don't run on forks
outputs:
storybook-url: ${{ steps.publish.outputs.storybookUrl }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # 👈 Required to retrieve git history (https://www.chromatic.com/docs/github-actions)
- name: Install pnpm
uses: pnpm/action-setup@v2
with:
version: 8.x.x
- name: Set up Node.js
uses: actions/setup-node@v3
with:
node-version: 18
cache: pnpm
- name: Install dependencies and Chromatic
run: pnpm i -D chromatic
- name: Publish to Chromatic
uses: chromaui/action@v1
id: publish
with:
token: ${{ secrets.GITHUB_TOKEN }}
# 👇 Chromatic projectToken, refer to the manage page to obtain it.
projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
visual-regression:
name: Visual regression tests
runs-on: ubuntu-latest
timeout-minutes: 30
container:
image: mcr.microsoft.com/playwright:v1.32.2 # Same as @storybook/[email protected]'s dependency
strategy:
fail-fast: false
matrix:
browser: ['chromium', 'webkit']
shard: [1, 2]
env:
SHARD_COUNT: '2'
CYPRESS_INSTALL_BINARY: '0'
NODE_OPTIONS: --max-old-space-size=6144
OPT_OUT_CAPTURE: 1
outputs:
# The below have to be manually listed unfortunately, as GitHub Actions doesn't allow matrix-dependent outputs
chromium-1-added: ${{ steps.diff.outputs.chromium-1-added }}
chromium-1-modified: ${{ steps.diff.outputs.chromium-1-modified }}
chromium-1-deleted: ${{ steps.diff.outputs.chromium-1-deleted }}
chromium-1-total: ${{ steps.diff.outputs.chromium-1-total }}
chromium-1-commitHash: ${{ steps.commit-hash.outputs.chromium-1-commitHash }}
chromium-2-added: ${{ steps.diff.outputs.chromium-2-added }}
chromium-2-modified: ${{ steps.diff.outputs.chromium-2-modified }}
chromium-2-deleted: ${{ steps.diff.outputs.chromium-2-deleted }}
chromium-2-total: ${{ steps.diff.outputs.chromium-2-total }}
chromium-2-commitHash: ${{ steps.commit-hash.outputs.chromium-2-commitHash }}
webkit-1-added: ${{ steps.diff.outputs.webkit-1-added }}
webkit-1-modified: ${{ steps.diff.outputs.webkit-1-modified }}
webkit-1-deleted: ${{ steps.diff.outputs.webkit-1-deleted }}
webkit-1-total: ${{ steps.diff.outputs.webkit-1-total }}
webkit-1-commitHash: ${{ steps.commit-hash.outputs.webkit-1-commitHash }}
webkit-2-added: ${{ steps.diff.outputs.webkit-2-added }}
webkit-2-modified: ${{ steps.diff.outputs.webkit-2-modified }}
webkit-2-deleted: ${{ steps.diff.outputs.webkit-2-deleted }}
webkit-2-total: ${{ steps.diff.outputs.webkit-2-total }}
webkit-2-commitHash: ${{ steps.commit-hash.outputs.webkit-2-commitHash }}
firefox-1-added: ${{ steps.diff.outputs.firefox-1-added }}
firefox-1-modified: ${{ steps.diff.outputs.firefox-1-modified }}
firefox-1-deleted: ${{ steps.diff.outputs.firefox-1-deleted }}
firefox-1-total: ${{ steps.diff.outputs.firefox-1-total }}
firefox-1-commitHash: ${{ steps.commit-hash.outputs.firefox-1-commitHash }}
firefox-2-added: ${{ steps.diff.outputs.firefox-2-added }}
firefox-2-modified: ${{ steps.diff.outputs.firefox-2-modified }}
firefox-2-deleted: ${{ steps.diff.outputs.firefox-2-deleted }}
firefox-2-total: ${{ steps.diff.outputs.firefox-2-total }}
firefox-2-commitHash: ${{ steps.commit-hash.outputs.firefox-2-commitHash }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
# Use PostHog Bot token when not on forks to enable proper snapshot updating
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
- name: Install pnpm
uses: pnpm/action-setup@v2
with:
version: 8.x.x
- name: Set up Node.js
uses: buildjet/setup-node@v3
with:
node-version: 18
cache: pnpm
- name: Install package.json dependencies with pnpm
run: pnpm install --frozen-lockfile
- name: Install CI utilities with pnpm
run: pnpm install http-server wait-on
- name: Build Storybook
run: pnpm build-storybook --quiet # Silence since progress logging results in a massive wall of spam
- name: Serve Storybook in the background
run: |
retries=3
while [ $retries -gt 0 ]; do
pnpm exec http-server storybook-static --port 6006 --silent &
if pnpm wait-on http://127.0.0.1:6006 --timeout 15; then
break
fi
retries=$((retries-1))
echo "Failed to serve Storybook, retrying... ($retries retries left)"
done
- name: Run @storybook/test-runner
env:
# Solving this bug by overriding $HOME: https://github.com/microsoft/playwright/issues/6500
HOME: /root
# Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
STORYBOOK_SKIP_TAGS: 'test-skip,test-skip-${{ matrix.browser }}'
run: |
pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT
- name: Run @playwright/test (legacy, Chromium-only)
if: matrix.browser == 'chromium' && matrix.shard == 1
env:
# Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
run: |
pnpm test:visual-regression:legacy:ci:$VARIANT
- name: Archive failure screenshots
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: failure-screenshots-${{ matrix.browser }}
path: frontend/__snapshots__/__failures__/
- name: Count and optimize updated snapshots
id: diff
# Skip on forks
if: github.event.pull_request.head.repo.full_name == github.repository
run: |
git config --global --add safe.directory '*' # Calm git down about file ownership
git diff --name-status frontend/__snapshots__/ # For debugging
ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l)
MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l)
DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l)
TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l)
# If added or modified, run OptiPNG
if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then
echo "Snapshots updated ($ADDED new, $MODIFIED changed), running OptiPNG"
apt update && apt install -y optipng
optipng -clobber -o4 -strip all
# we don't want to _always_ run OptiPNG
# so, we run it after checking for a diff
# but, the files we diffed might then be changed by OptiPNG
# and as a result they might no longer be different...
# we check again
git diff --name-status frontend/__snapshots__/ # For debugging
ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l)
MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l)
DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l)
TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l)
if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then
echo "Snapshots updated ($ADDED new, $MODIFIED changed), _even after_ running OptiPNG"
git add frontend/__snapshots__/ playwright/
fi
fi
echo "${{ matrix.browser }}-${{ matrix.shard }}-added=$ADDED" >> $GITHUB_OUTPUT
echo "${{ matrix.browser }}-${{ matrix.shard }}-modified=$MODIFIED" >> $GITHUB_OUTPUT
echo "${{ matrix.browser }}-${{ matrix.shard }}-deleted=$DELETED" >> $GITHUB_OUTPUT
echo "${{ matrix.browser }}-${{ matrix.shard }}-total=$TOTAL" >> $GITHUB_OUTPUT
- name: Commit updated snapshots
uses: EndBug/add-and-commit@v9
if: github.event.pull_request.head.repo.full_name == github.repository
id: commit
with:
add: '["frontend/__snapshots__/", "playwright/"]'
message: 'Update UI snapshots for `${{ matrix.browser }}` (${{ matrix.shard }})'
pull: --rebase --autostash # Make sure we're up to date with other browsers' updates
default_author: github_actions
github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
- name: Add commit hash to outputs, including browser name
id: commit-hash
if: steps.commit.outputs.pushed == 'true'
run: echo "${{ matrix.browser }}-${{ matrix.shard }}-commitHash=${{ steps.commit.outputs.commit_long_sha }}" >> $GITHUB_OUTPUT
visual-regression-summary:
name: Summarize visual regression tests
runs-on: ubuntu-latest
timeout-minutes: 5
needs: visual-regression
if: always() # Run even if visual-regression fails for one (or more) of the browsers
steps:
- name: Post comment about updated snapshots
if: github.event.pull_request.head.repo.full_name == github.repository
uses: actions/github-script@v6
with:
github-token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
script: |
const BROWSERS = ['chromium', 'webkit']
const diffJobOutputs = ${{ toJson(needs.visual-regression.outputs) }}
const summaryDiff = { total: 0, added: 0, modified: 0, deleted: 0 }
const diffByBrowser = Object.fromEntries(BROWSERS.map(browser => [browser, {
total: 0, added: 0, modified: 0, deleted: 0, commitHashes: []
}]))
for (const [key, rawValue] of Object.entries(diffJobOutputs)) {
// Split e.g. 'chromium-1-commitHash' into ['chromium', '1' 'commitHash']
const [browser, shardNumber, diffKey] = key.split('-')
// Sum up the counts - but not the commit hash
if (diffKey === 'commitHash') {
diffByBrowser[browser].commitHashes.push([parseInt(shardNumber), rawValue])
} else {
const value = parseInt(rawValue)
diffByBrowser[browser][diffKey] += value
summaryDiff[diffKey] += value
}
}
for (const browser of BROWSERS) {
if (diffByBrowser[browser]?.total === undefined) {
diffByBrowser[browser] = null // Null means failure
}
}
if (summaryDiff.total === 0) {
console.log('No changes were made, skipping comment')
return
}
const diffByBrowserDisplay = Object.entries(diffByBrowser).map(([browser, diff]) => {
if (!diff) {
return `- \`${browser}\`: failed`
}
const { added: a, modified: m, deleted: d, commitHashes } = diff
const b = a + m + d > 0 ? '**' : '' // Bold list item if there were changes
let extraInfo = ''
if (b) {
const commitInfo = commitHashes.map(
([shardNumber, commitHash]) =>
`[diff for shard ${shardNumber}](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${commitHash})`
).join(', ') || "wasn't pushed!"
extraInfo = ` (${commitInfo})`
}
return `- ${b}\`${browser}\`${b}: **${a}** added, **${m}** modified, **${d}** deleted${extraInfo}`
}).join('\n')
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: `## 📸 UI snapshots have been updated
**${summaryDiff.total}** snapshot changes in total. **${summaryDiff.added}** added, **${summaryDiff.modified}** modified, **${summaryDiff.deleted}** deleted:
${diffByBrowserDisplay}
Triggered by [this commit](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${{ github.sha }}).
👉 **[Review this PR's diff of snapshots.](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/files#:~:text=frontend/__snapshots__/)**`
})