chore: make visual regression less sensitive to differences (#17455)

* chore: make visual regression less sensitive to differences * Update UI snapshots for `chromium` (1) * git diff only after optipng has run * git diff only after optipng has run * Update UI snapshots for `chromium` (1) * Update .storybook/test-runner.ts * text * no need to run lint staged... we've already run optipng --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
PostHog · Sep 15, 2023 · 3908adb · 3908adb
1 parent 35b0adf
commit 3908adb
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 3 deletions.
diff --git a/.github/workflows/storybook-chromatic.yml b/.github/workflows/storybook-chromatic.yml
@@ -165,8 +165,24 @@ jobs:
                   if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then
                     echo "Snapshots updated ($ADDED new, $MODIFIED changed), running OptiPNG"
                     apt update && apt install -y optipng
-                    git add frontend/__snapshots__/ playwright/
-                    pnpm lint-staged
+                    optipng -clobber -o4 -strip all
+                    
+                    # we don't want to _always_ run OptiPNG
+                    # so, we run it after checking for a diff
+                    # but, the files we diffed might then be changed by OptiPNG
+                    # and as a result they might no longer be different...
+                    
+                    # we check again
+                    git diff --name-status frontend/__snapshots__/ # For debugging
+                    ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l)
+                    MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l)
+                    DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l)
+                    TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l)
+
+                    if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then
+                        echo "Snapshots updated ($ADDED new, $MODIFIED changed), _even after_ running OptiPNG"
+                        git add frontend/__snapshots__/ playwright/
+                    fi
                   fi
 
                   echo "${{ matrix.browser }}-${{ matrix.shard }}-added=$ADDED" >> $GITHUB_OUTPUT

diff --git a/.storybook/test-runner.ts b/.storybook/test-runner.ts
@@ -208,7 +208,8 @@ async function expectLocatorToMatchStorySnapshot(
         // Compare structural similarity instead of raw pixels - reducing false positives
         // See https://github.com/americanexpress/jest-image-snapshot#recommendations-when-using-ssim-comparison
         comparisonMethod: 'ssim',
-        failureThreshold: 0.0003,
+        // 0.01 would be a 1% difference
+        failureThreshold: 0.01,
         failureThresholdType: 'percent',
     })
 }