feat(ci): CI/CD workflows and composite actions to check test results (#1533)

.github/actions/cache-playwright/action.yml

@@ -0,0 +1,38 @@
+name: "Install and Cache Playwright"
+description: "Sets up a cache and/or installs Playwright and its binaries."
+
+inputs:
+ WORKSPACE_ROOT:
+ description: "The workspace root."
+ required: true
+
+runs:
+ using: "composite"
+ steps:
+ - name: Get installed Playwright version
+ shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ id: playwright-version
+ run: echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright' || 'latest')" >> $GITHUB_ENV
+
+ - name: Cache playwright binaries
+ uses: actions/cache@v4
+ id: playwright-cache
+ with:
+ path: |
+ ~/.cache/ms-playwright
+ key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
+
+ - shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: npm ci
+
+ - shell: bash
+ if: steps.playwright-cache.outputs.cache-hit != 'true'
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: npx playwright install --with-deps
+
+ - shell: bash
+ if: steps.playwright-cache.outputs.cache-hit != 'true'
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: npx playwright install-deps

.github/actions/prepare-docker/action.yml

@@ -0,0 +1,40 @@
+name: "Pull & Prepare Docker Workflow"
+description: "Sets name and tag, logs in to GHCR, and pulls Docker image."
+
+inputs:
+ REGISTRY:
+ description: "GitHub Registry"
+ required: true
+ ACTOR:
+ description: "GitHub Username"
+ required: true
+ GITHUB_TOKEN:
+ description: "GitHub Token"
+ required: true
+
+runs:
+ using: "composite"
+ steps:
+ - name: Set Name and Tag Vars
+ shell: bash
+ working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+ env:
+ name: "${{ env.BRANCH_NAME }}"
+ run: |
+ echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV}
+ echo "TAG=${name/\//-}" >> $GITHUB_ENV
+ echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV
+ echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')" >> $GITHUB_ENV
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.REGISTRY }}
+ username: ${{ inputs.ACTOR }}
+ password: ${{ inputs.GITHUB_TOKEN }}
+
+ - name: Pull Image
+ shell: bash
+ working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+ run: |
+ docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG

.github/actions/sanity-test-checks/action.yml

@@ -0,0 +1,64 @@
+name: "Validate Sanity Report"
+description: "Diff the sanity-test reports with a fixture and validate expected shape."
+
+inputs:
+ WORKSPACE_ROOT:
+ description: "Working Directory"
+ required: true
+ RUNNER:
+ description: "Test Runner"
+ required: true
+ FIXTURE:
+ description: "Test Report Fixture"
+ required: true
+
+runs:
+ using: "composite"
+ steps:
+ - name: "Validate ${{ inputs.RUNNER }} Report"
+ id: validate-report
+ continue-on-error: true
+ shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: |
+ set +e
+ FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}"
+ TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' $FIXTURE) <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
+
+ echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
+ if [[ "$TEST_RESULT" != "" ]]; then
+ echo "# ❎ ${{ inputs.RUNNER }} Sanity Different" >> $GITHUB_STEP_SUMMARY
+ echo '```diff' >> $GITHUB_STEP_SUMMARY
+ echo "${TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+ echo '```' >> $GITHUB_STEP_SUMMARY
+ else
+ echo "# ✅ ${{ inputs.RUNNER }} Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
+ fi
+
+ - name: "Full Sanity Report Diff"
+ id: diff
+ continue-on-error: true
+ shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: |
+ set +e
+ FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' $FIXTURE) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+ echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+ echo "<details>" >> $GITHUB_STEP_SUMMARY
+ echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo '```diff' >> $GITHUB_STEP_SUMMARY
+ echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+ echo '```' >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+ - name: "Report Validation Outcome"
+ shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: |
+ if [[ "$TEST_RESULT" != "" ]]; then
+ exit 1
+ else
+ exit 0
+ fi

.github/actions/setup-base/action.yml

@@ -0,0 +1,23 @@
+name: "Setup Base Workflow"
+description: "Checks out the BackstopJS repo at the calling or current branch with shallow history. Pulls Docker image by branch tag. Installs dependencies. Optionally sets up interactive remote debug shell."
+
+outputs:
+ WORKSPACE_ROOT:
+ description: "Export the root workspace for all workflows."
+ value: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}
+
+runs:
+ using: "composite"
+ steps:
+ - name: Set workspace root
+ id: workspace_root
+ shell: bash
+ run: |
+ WORKSPACE_ROOT=$GITHUB_WORKSPACE/repo
+ echo "WORKSPACE_ROOT=$WORKSPACE_ROOT" | tee -a $GITHUB_OUTPUT
+
+ - name: Checkout to workspace
+ shell: bash
+ run: |
+ git clone -b ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} https://github.com/${{ github.repository }}.git ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}
+ ls ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}

.github/actions/setup-node/action.yml

@@ -0,0 +1,22 @@
+name: "Setup Node and Dependencies"
+description: "Sets up Node, npm caching, and installs dependencies"
+
+inputs:
+ WORKSPACE_ROOT:
+ description: "The workspace root."
+ required: true
+
+runs:
+ using: "composite"
+ steps:
+ - name: ⬢ Setup Node & Cache
+ uses: actions/setup-node@v4
+ with:
+ node-version: ${{ env.NODE_VERSION }}
+ cache: "npm"
+ cache-dependency-path: ${{ inputs.WORKSPACE_ROOT }}/package-lock.json
+
+ - name: ↧ Install
+ shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: npm ci

.github/actions/smoke-test-checks/action.yml

@@ -0,0 +1,64 @@
+name: "Validate Smoke Report"
+description: "Diff the smoke-test reports with a fixture and validate expected shape."
+
+inputs:
+ WORKSPACE_ROOT:
+ description: "Working Directory"
+ required: true
+ RUNNER:
+ description: "Test Runner"
+ required: true
+ FIXTURE:
+ description: "Test Report Fixture"
+ required: true
+
+runs:
+ using: "composite"
+ steps:
+ - name: "Validate ${{ inputs.RUNNER }} Report"
+ id: validate
+ continue-on-error: true
+ shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: |
+ set +e
+ FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}"
+ TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del( .tests[].pair.diff, .tests[].pair.diffImage)' $FIXTURE) <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del( .tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
+
+ echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
+ if [[ "$TEST_RESULT" != "" ]]; then
+ echo "# ❎ ${{ inputs.RUNNER }} Report Different" >> $GITHUB_STEP_SUMMARY
+ echo '```diff' >> $GITHUB_STEP_SUMMARY
+ echo "$TEST_RESULT" >> $GITHUB_STEP_SUMMARY
+ echo '```'
+ else
+ echo "# ✅ ${{ inputs.RUNNER }} Report Validated" >> $GITHUB_STEP_SUMMARY
+ fi
+
+ - name: "Full Smoke Report Diff"
+ id: diff
+ continue-on-error: true
+ shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: |
+ set +e
+ FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' $FIXTURE) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+ echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+ echo "<details>" >> $GITHUB_STEP_SUMMARY
+ echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo '```diff' >> $GITHUB_STEP_SUMMARY
+ echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+ echo '```' >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+ - name: "Report Validation Outcome"
+ shell: bash
+ working-directory: ${{ inputs.WORKSPACE_ROOT }}
+ run: |
+ if [[ "$TEST_RESULT" != "" ]]; then
+ exit 1
+ else
+ exit 0
+ fi

.github/workflows/docker-build.yml

@@ -38,7 +38,7 @@ jobs:
  echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV
 
  - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+ uses: docker/login-action@v3
  with:
  registry: ${{ env.REGISTRY }}
  username: ${{ github.actor }}

.github/workflows/docker-sanity-test-checks.yml

@@ -0,0 +1,99 @@
+name: Docker Sanity Test Checks
+
+on:
+ workflow_dispatch:
+ workflow_call:
+
+permissions:
+ actions: write
+ checks: write
+ contents: write
+ pull-requests: write
+ packages: write
+
+env:
+ BRANCH_NAME: ${{ github.event.pull_request.head_ref || github.event.pull_request.head.ref_name || github.head_ref || github.ref_name }}
+ NODE_VERSION: 20
+ REGISTRY: ghcr.io
+ IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+ sanity-test-puppeteer:
+ name: 🤪 Puppeteer
+ runs-on: ubuntu-latest
+
+ permissions:
+ contents: read
+ packages: write
+
+ steps:
+ - name: Checkout actions
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }}
+ sparse-checkout: |
+ .github/actions
+ path: actions
+
+ - uses: ./actions/.github/actions/setup-base
+ id: base
+
+ - uses: ./actions/.github/actions/prepare-docker
+ with:
+ REGISTRY: ${{ env.REGISTRY }}
+ ACTOR: ${{ github.actor }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: "𓋏 Run `backstop test` in Docker"
+ continue-on-error: true
+ shell: bash
+ working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+ run: |
+ set +e
+ cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test
+
+ - name: "Validate Puppeteer Docker Test Results"
+ uses: ./actions/.github/actions/sanity-test-checks
+ with:
+ WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+ RUNNER: "Puppeteer"
+ FIXTURE: "sanity-test-docker.json"
+
+ sanity-test-playwright:
+ name: 🤪 Playwright
+ runs-on: ubuntu-latest
+
+ permissions:
+ contents: read
+ packages: write
+
+ steps:
+ - name: Checkout actions
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }}
+ sparse-checkout: |
+ .github/actions
+ path: actions
+
+ - uses: ./actions/.github/actions/setup-base
+ id: base
+
+ - uses: ./actions/.github/actions/prepare-docker
+ with:
+ REGISTRY: ${{ env.REGISTRY }}
+ ACTOR: ${{ github.actor }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: "🎭 Run `backstop test --confg=playwright` in Docker"
+ continue-on-error: true
+ run: |
+ set +e
+ cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose --foreground-scripts i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=playwright"
+
+ - name: "Validate Playwright Docker Test Results"
+ uses: ./actions/.github/actions/sanity-test-checks
+ with:
+ WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+ RUNNER: "Playwright"
+ FIXTURE: "sanity-test-playwright-docker.json"

.github/workflows/docker-sanity-test.yml

@@ -43,7 +43,7 @@ jobs:
  echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')" >> $GITHUB_ENV
 
  - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+ uses: docker/login-action@v3
  with:
  registry: ${{ env.REGISTRY }}
  username: ${{ github.actor }}
@@ -91,7 +91,7 @@ jobs:
  echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')" >> $GITHUB_ENV
 
  - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+ uses: docker/login-action@v3
  with:
  registry: ${{ env.REGISTRY }}
  username: ${{ github.actor }}