diff --git a/.github/workflows/ci-app-pr-environment-update.yml b/.github/workflows/ci-app-pr-environment-checks.yml similarity index 94% rename from .github/workflows/ci-app-pr-environment-update.yml rename to .github/workflows/ci-app-pr-environment-checks.yml index 67724bc2..2bfc9397 100644 --- a/.github/workflows/ci-app-pr-environment-update.yml +++ b/.github/workflows/ci-app-pr-environment-checks.yml @@ -1,4 +1,4 @@ -name: CI App PR Environment Update +name: CI App PR Environment Checks on: workflow_dispatch: inputs: diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 00000000..267ac0cb --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,55 @@ +name: E2E Tests + +on: + workflow_call: + inputs: + service_endpoint: + required: true + type: string + app_name: + required: false + type: string + +jobs: + e2e: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + runs-on: ubuntu-latest + + env: + BASE_URL: ${{ inputs.service_endpoint }} + APP_NAME: ${{ inputs.app_name }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libwoff1 libopus0 libvpx7 libevent-2.1-7 libopus0 libgstreamer1.0-0 \ + libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0 libharfbuzz-icu0 libhyphen0 \ + libenchant-2-2 libflite1 libgles2 libx264-dev + + - name: Install Node.js dependencies + run: npm ci + working-directory: ./e2e + + - name: Install Playwright browsers + run: make e2e-setup-ci + + - name: Run e2e tests + run: make e2e-test APP_NAME=${{ inputs.app_name }} BASE_URL=${{ inputs.service_endpoint }} + env: + BASE_URL: ${{ inputs.service_endpoint }} + APP_NAME: ${{ inputs.app_name }} + + - name: Upload Playwright report + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: ./e2e/playwright-report diff --git a/.github/workflows/pr-environment-update.yml b/.github/workflows/pr-environment-update.yml index 585524e9..30f1a4a9 100644 --- a/.github/workflows/pr-environment-update.yml +++ b/.github/workflows/pr-environment-update.yml @@ -35,6 +35,9 @@ jobs: concurrency: pr-environment-${{ inputs.pr_number }} + outputs: + service_endpoint: ${{ steps.update-environment.outputs.service_endpoint }} + steps: - uses: actions/checkout@v4 @@ -51,6 +54,19 @@ jobs: environment: ${{ inputs.environment }} - name: Update environment - run: ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}" + id: update-environment + run: | + ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}" + service_endpoint=$(terraform -chdir="infra/${{ inputs.app_name }}/service" output -raw service_endpoint) + echo "service_endpoint=${service_endpoint}" + echo "service_endpoint=${service_endpoint}" >> "$GITHUB_OUTPUT" env: GH_TOKEN: ${{ github.token }} + + e2e-tests: + name: Run E2E Tests + needs: [update] + uses: ./.github/workflows/e2e-tests.yml + with: + service_endpoint: ${{ needs.update.outputs.service_endpoint }} + app_name: ${{ inputs.app_name }} diff --git a/Makefile b/Makefile index a4c2c296..5b1840bc 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,9 @@ __check_defined = \ release-image-name \ release-image-tag \ release-publish \ - release-run-database-migrations - - + release-run-database-migrations \ + e2e-setup \ + e2e-test infra-set-up-account: ## Configure and create resources for current AWS profile and save tfbackend file to infra/accounts/$ACCOUNT_NAME.ACCOUNT_ID.s3.tfbackend @:$(call check_defined, ACCOUNT_NAME, human readable name for account e.g. "prod" or the AWS account alias) @@ -217,6 +217,23 @@ release-image-name: ## Prints the image name of the release image release-image-tag: ## Prints the image tag of the release image @echo $(IMAGE_TAG) +############################## +## End-to-end (E2E) Testing ## +############################## + +e2e-setup: ## Setup end-to-end tests + @cd e2e && npm install + @cd e2e && npx playwright install --with-deps + +e2e-setup-ci: ## Install dependencies and Playwright browsers + cd e2e && npx playwright install --with-deps + +e2e-test: ## Run end-to-end tests + # make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 + @:$(call check_defined, APP_NAME, ...) + @:$(call check_defined, BASE_URL, ...) + @cd e2e && cd $(APP_NAME) && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npx playwright test $(E2E_ARGS) + ######################## ## Scripts and Helper ## ######################## diff --git a/docs/e2e/e2e-checks.md b/docs/e2e/e2e-checks.md new file mode 100644 index 00000000..77e266a7 --- /dev/null +++ b/docs/e2e/e2e-checks.md @@ -0,0 +1,76 @@ +# End-to-End (E2E) Tests + +## Overview + +This repository uses [Playwright](https://playwright.dev/) to perform end-to-end (E2E) tests. The tests can be run locally, but also run on [Pull Request preview environments](../infra/pull-request-environments.md). This ensures that any new code changes are validated through E2E tests before being merged. + +## Folder Structure +In order to support e2e for multiple apps, the folder structure will include a base playwright config (`./e2e/playwright.config.js`), and app-specific derived playwright config that override the base config. See the example folder structure below: +``` +- e2e + - playwright.config.js + - app/ + - playwright.config.js + - tests/ + - index.spec.js + - app2/ + - playwright.config.js + - tests/ + - index.spec.js +``` + +Some highlights: +- By default, the base config is defined to run on a minimal browser-set (desktop and mobile chrome) +- Snapshots will be output locally or in the artifacts of the CI job +- HTML reports are output to the `playwright-report` folder +- Parallelism limited on CI to ensure stable execution +- Accessibility testing can be performed using the `@axe-core/playwright` package (https://playwright.dev/docs/accessibility-testing) + + +## Running Locally + +### Running Locally From the Root Directory + +Make targets are setup to easily pass in a particular app name and URL to run tests against + +``` +make e2e-setup # install playwright deps +make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 # run tests on a particular app +``` + +### Running Locally From the `./e2e` Directory + +If you prefer to run package.json run scripts, you can do so by creating a `./e2e/.env` file with an `APP_NAME` and `BASE_URL` + +``` +cd e2e + +# Create .env file with BASE_URL and APP_NAME +echo "BASE_URL=http://127.0.0.1:3000" > .env +echo "APP_NAME=your-app-name" >> .env + +npm install +npm run e2e-test +``` + +### PR Environments + +The E2E tests are triggered in PR preview environments on each PR update. For more information on how PR environments work, please refer to [PR Environments Documentation](../infra/pull-request-environments.md). + +### Workflows + +The following workflows trigger E2E tests: +- [PR Environment Update](../../.github/workflows/pr-environment-update.yml) +- [E2E Tests Workflow](../../.github/workflows/e2e-tests.yml) + +The [E2E Tests Workflow](../../.github/workflows/e2e-tests.yml) takes a `service_endpoint` URL and an `app_name` to run the tests against specific configurations for your app. + +## Configuration + +The E2E tests are configured using the following files: +- [Base Configuration](../../e2e/playwright.config.js) +- [App-specific Configuration](../../e2e/app/playwright.config.js) + +The app-specific configuration files extend the common base configuration. + +By default when running `make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 ` - you don't necessarily need to pass an `BASE_URL` since the default is defined in the app-specific playwright config (`./e2e/app/playwright.config.js`). diff --git a/e2e/.env.example b/e2e/.env.example new file mode 100644 index 00000000..80dd751a --- /dev/null +++ b/e2e/.env.example @@ -0,0 +1,2 @@ +BASE_URL=http://127.0.0.1:3000 +APP_NAME=app diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 00000000..68f91e36 --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ +*.png* diff --git a/e2e/app/playwright.config.js b/e2e/app/playwright.config.js new file mode 100644 index 00000000..6beb2d89 --- /dev/null +++ b/e2e/app/playwright.config.js @@ -0,0 +1,19 @@ +import { defineConfig, devices } from '@playwright/test'; + +import baseConfig from '../playwright.config'; + +export default defineConfig({ + ...baseConfig, + use: { + ...baseConfig.use, + baseUrl: baseConfig.use.baseUrl || "localhost:3000" + }, + projects: [ + ...baseConfig.projects, + // add Safari for derived app config + { + name: "webkit", + use: { ...devices["Desktop Safari"] }, + }, + ], +}); diff --git a/e2e/app/tests/index.spec.js b/e2e/app/tests/index.spec.js new file mode 100644 index 00000000..51c07df4 --- /dev/null +++ b/e2e/app/tests/index.spec.js @@ -0,0 +1,31 @@ +const { test, expect } = require('@playwright/test'); + +import AxeBuilder from '@axe-core/playwright'; + +test.describe('Generic Webpage Tests', () => { + test('should load the webpage successfully', async ({ page }) => { + const response = await page.goto('/'); + const title = await page.title(); + await expect(response.status()).toBe(200); + }); + + test('should take a screenshot of the webpage', async ({ page }) => { + await page.goto('/'); + await page.screenshot({ path: 'example-screenshot.png', fullPage: true }); + }); + + // https://playwright.dev/docs/accessibility-testing + test('should not have any automatically detectable accessibility issues', async ({ page }) => { + await page.goto('/'); + const accessibilityScanResults = await new AxeBuilder({ page }).analyze(); + expect(accessibilityScanResults.violations).toEqual([]); + }); + + // Example test of finding a an html element on the index/home page + // test('should check for an element to be visible', async ({ page }) => { + // await page.goto('/'); + // const element = page.locator('h1'); + // await expect(element).toBeVisible(); + // }); + +}); diff --git a/e2e/package-lock.json b/e2e/package-lock.json new file mode 100644 index 00000000..1134911c --- /dev/null +++ b/e2e/package-lock.json @@ -0,0 +1,123 @@ +{ + "name": "e2e", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "e2e", + "version": "1.0.0", + "dependencies": { + "@axe-core/playwright": "^4.9.1", + "dotenv": "^16.4.5" + }, + "devDependencies": { + "@playwright/test": "^1.45.1", + "@types/node": "^20.14.10" + } + }, + "node_modules/@axe-core/playwright": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.9.1.tgz", + "integrity": "sha512-8m4WZbZq7/aq7ZY5IG8GqV+ZdvtGn/iJdom+wBg+iv/3BAOBIfNQtIu697a41438DzEEyptXWmC3Xl5Kx/o9/g==", + "dependencies": { + "axe-core": "~4.9.1" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.45.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.45.1.tgz", + "integrity": "sha512-Wo1bWTzQvGA7LyKGIZc8nFSTFf2TkthGIFBR+QVNilvwouGzFd4PYukZe3rvf5PSqjHi1+1NyKSDZKcQWETzaA==", + "dev": true, + "dependencies": { + "playwright": "1.45.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "20.14.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.10.tgz", + "integrity": "sha512-MdiXf+nDuMvY0gJKxyfZ7/6UFsETO7mGKF54MVD/ekJS6HdFtpZFBgrh6Pseu64XTb2MLyFPlbW6hj8HYRQNOQ==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/axe-core": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.9.1.tgz", + "integrity": "sha512-QbUdXJVTpvUTHU7871ppZkdOLBeGUKBQWHkHrvN2V9IQWGMt61zf3B45BtzjxEJzYuj0JBjBZP/hmYS/R9pmAw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.45.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.45.1.tgz", + "integrity": "sha512-Hjrgae4kpSQBr98nhCj3IScxVeVUixqj+5oyif8TdIn2opTCPEzqAqNMeK42i3cWDCVu9MI+ZsGWw+gVR4ISBg==", + "dev": true, + "dependencies": { + "playwright-core": "1.45.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.45.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.45.1.tgz", + "integrity": "sha512-LF4CUUtrUu2TCpDw4mcrAIuYrEjVDfT1cHbJMfwnE2+1b8PZcFzPNgvZCvq2JfQ4aTjRCCHw5EJ2tmr2NSzdPg==", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + } + } +} diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 00000000..4b8decad --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,17 @@ +{ + "name": "e2e", + "version": "1.0.0", + "scripts": { + "e2e-setup": "npx playwright install", + "e2e-test": "sh -c 'source .env && npx playwright test --config $APP_NAME/playwright.config.js'", + "e2e-test:ui": "npx playwright test --ui" + }, + "devDependencies": { + "@playwright/test": "^1.45.1", + "@types/node": "^20.14.10" + }, + "dependencies": { + "@axe-core/playwright": "^4.9.1", + "dotenv": "^16.4.5" + } +} diff --git a/e2e/playwright.config.js b/e2e/playwright.config.js new file mode 100644 index 00000000..1ee87f60 --- /dev/null +++ b/e2e/playwright.config.js @@ -0,0 +1,50 @@ +// Load environment variables from .env file if it exists +import * as dotenv from 'dotenv'; + +import { defineConfig, devices } from "@playwright/test"; + +dotenv.config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + // Timeout for each test in milliseconds + timeout: 20000, + testDir: "./tests", // Ensure this points to the correct test directory + // Run tests in files in parallel + fullyParallel: true, + // Fail the build on CI if you accidentally left test.only in the source code. + forbidOnly: !!process.env.CI, + // Retry on CI only + retries: process.env.CI ? 2 : 0, + // Opt out of parallel tests on CI. + workers: process.env.CI ? 1 : undefined, + // Reporter to use. See https://playwright.dev/docs/test-reporters + reporter: "html", + // Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. + use: { + // Base URL to use in actions like `await page.goto('/')`. + baseURL: process.env.BASE_URL, + + // Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer + trace: "on-first-retry", + screenshot: "on", + video: "on-first-retry", + }, + + // Configure projects for major browsers + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + + // Test against mobile viewports. + { + name: "Mobile Chrome", + use: { ...devices["Pixel 7"] }, + }, + ], + +});