[Issue #358]: Add e2e tests against PR preview env

.github/workflows/ci-app-pr-environment-update.yml → .github/workflows/ci-app-pr-environment-checks.yml

@@ -1,4 +1,4 @@
-name: CI App PR Environment Update
+name: CI App PR Environment Checks
 on:
  workflow_dispatch:
  inputs:

.github/workflows/e2e-tests.yml

@@ -0,0 +1,55 @@
+name: E2E Tests
+
+on:
+ workflow_call:
+ inputs:
+ service_endpoint:
+ required: true
+ type: string
+ app_name:
+ required: false
+ type: string
+
+jobs:
+ e2e:
+ name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise
+ runs-on: ubuntu-latest
+
+ env:
+ BASE_URL: ${{ inputs.service_endpoint }}
+ APP_NAME: ${{ inputs.app_name }}
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+
+ - name: Install system dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libwoff1 libopus0 libvpx7 libevent-2.1-7 libopus0 libgstreamer1.0-0 \
+ libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0 libharfbuzz-icu0 libhyphen0 \
+ libenchant-2-2 libflite1 libgles2 libx264-dev
+
+ - name: Install Node.js dependencies
+ run: npm ci
+ working-directory: ./e2e
+
+ - name: Install Playwright browsers
+ run: make e2e-setup-ci
+
+ - name: Run e2e tests
+ run: make e2e-test APP_NAME=${{ inputs.app_name }} BASE_URL=${{ inputs.service_endpoint }}
+ env:
+ BASE_URL: ${{ inputs.service_endpoint }}
+ APP_NAME: ${{ inputs.app_name }}
+
+ - name: Upload Playwright report
+ uses: actions/upload-artifact@v4
+ with:
+ name: playwright-report
+ path: ./e2e/playwright-report

.github/workflows/pr-environment-update.yml

@@ -35,6 +35,9 @@ jobs:
 
  concurrency: pr-environment-${{ inputs.pr_number }}
 
+ outputs:
+ service_endpoint: ${{ steps.update-environment.outputs.service_endpoint }}
+
  steps:
  - uses: actions/checkout@v4
 
@@ -51,6 +54,19 @@ jobs:
  environment: ${{ inputs.environment }}
 
  - name: Update environment
- run: ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}"
+ id: update-environment
+ run: |
+ ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}"
+ service_endpoint=$(terraform -chdir="infra/${{ inputs.app_name }}/service" output -raw service_endpoint)
+ echo "service_endpoint=${service_endpoint}"
+ echo "service_endpoint=${service_endpoint}" >> "$GITHUB_OUTPUT"
  env:
  GH_TOKEN: ${{ github.token }}
+
+ e2e-tests:
+ name: Run E2E Tests
+ needs: [update]
+ uses: ./.github/workflows/e2e-tests.yml
+ with:
+ service_endpoint: ${{ needs.update.outputs.service_endpoint }}
+ app_name: ${{ inputs.app_name }}

Makefile

@@ -58,9 +58,9 @@ __check_defined = \
  release-image-name \
  release-image-tag \
  release-publish \
- release-run-database-migrations
-
-
+ release-run-database-migrations \
+ e2e-setup \
+ e2e-test
 
 infra-set-up-account: ## Configure and create resources for current AWS profile and save tfbackend file to infra/accounts/$ACCOUNT_NAME.ACCOUNT_ID.s3.tfbackend
  @:$(call check_defined, ACCOUNT_NAME, human readable name for account e.g. "prod" or the AWS account alias)
@@ -217,6 +217,23 @@ release-image-name: ## Prints the image name of the release image
 release-image-tag: ## Prints the image tag of the release image
  @echo $(IMAGE_TAG)
 
+##############################
+## End-to-end (E2E) Testing ##
+##############################
+
+e2e-setup: ## Setup end-to-end tests
+ @cd e2e && npm install
+ @cd e2e && npx playwright install --with-deps
+
+e2e-setup-ci: ## Install dependencies and Playwright browsers
+ cd e2e && npx playwright install --with-deps
+
+e2e-test: ## Run end-to-end tests
+ # make e2e-test APP_NAME=app BASE_URL=http://localhost:3000
+ @:$(call check_defined, APP_NAME, ...)
+ @:$(call check_defined, BASE_URL, ...)
+ @cd e2e && cd $(APP_NAME) && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npx playwright test $(E2E_ARGS)
+
 ########################
 ## Scripts and Helper ##
 ########################

docs/e2e/e2e-checks.md

@@ -0,0 +1,76 @@
+# End-to-End (E2E) Tests
+
+## Overview
+
+This repository uses [Playwright](https://playwright.dev/) to perform end-to-end (E2E) tests. The tests can be run locally, but also run on [Pull Request preview environments](../infra/pull-request-environments.md). This ensures that any new code changes are validated through E2E tests before being merged.
+
+## Folder Structure
+In order to support e2e for multiple apps, the folder structure will include a base playwright config (`./e2e/playwright.config.js`), and app-specific derived playwright config that override the base config. See the example folder structure below:
+```
+- e2e
+ - playwright.config.js
+ - app/
+ - playwright.config.js
+ - tests/
+ - index.spec.js
+ - app2/
+ - playwright.config.js
+ - tests/
+ - index.spec.js
+```
+
+Some highlights:
+- By default, the base config is defined to run on a minimal browser-set (desktop and mobile chrome)
+- Snapshots will be output locally or in the artifacts of the CI job
+- HTML reports are output to the `playwright-report` folder
+- Parallelism limited on CI to ensure stable execution
+- Accessibility testing can be performed using the `@axe-core/playwright` package (https://playwright.dev/docs/accessibility-testing)
+
+
+## Running Locally
+
+### Running Locally From the Root Directory
+
+Make targets are setup to easily pass in a particular app name and URL to run tests against
+
+```
+make e2e-setup # install playwright deps
+make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 # run tests on a particular app
+```
+
+### Running Locally From the `./e2e` Directory
+
+If you prefer to run package.json run scripts, you can do so by creating a `./e2e/.env` file with an `APP_NAME` and `BASE_URL`
+
+```
+cd e2e
+
+# Create .env file with BASE_URL and APP_NAME
+echo "BASE_URL=http://127.0.0.1:3000" > .env
+echo "APP_NAME=your-app-name" >> .env
+
+npm install
+npm run e2e-test
+```
+
+### PR Environments
+
+The E2E tests are triggered in PR preview environments on each PR update. For more information on how PR environments work, please refer to [PR Environments Documentation](../infra/pull-request-environments.md).
+
+### Workflows
+
+The following workflows trigger E2E tests:
+- [PR Environment Update](../../.github/workflows/pr-environment-update.yml)
+- [E2E Tests Workflow](../../.github/workflows/e2e-tests.yml)
+
+The [E2E Tests Workflow](../../.github/workflows/e2e-tests.yml) takes a `service_endpoint` URL and an `app_name` to run the tests against specific configurations for your app.
+
+## Configuration
+
+The E2E tests are configured using the following files:
+- [Base Configuration](../../e2e/playwright.config.js)
+- [App-specific Configuration](../../e2e/app/playwright.config.js)
+
+The app-specific configuration files extend the common base configuration.
+
+By default when running `make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 ` - you don't necessarily need to pass an `BASE_URL` since the default is defined in the app-specific playwright config (`./e2e/app/playwright.config.js`).

e2e/.env.example

@@ -0,0 +1,2 @@
+BASE_URL=http://127.0.0.1:3000
+APP_NAME=app

e2e/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+*.png*

e2e/app/playwright.config.js

@@ -0,0 +1,19 @@
+import { defineConfig, devices } from '@playwright/test';
+
+import baseConfig from '../playwright.config';
+
+export default defineConfig({
+ ...baseConfig,
+ use: {
+ ...baseConfig.use,
+ baseUrl: baseConfig.use.baseUrl || "localhost:3000"
+ },
+ projects: [
+ ...baseConfig.projects,
+ // add Safari for derived app config
+ {
+ name: "webkit",
+ use: { ...devices["Desktop Safari"] },
+ },
+ ],
+});

e2e/app/tests/index.spec.js

@@ -0,0 +1,31 @@
+const { test, expect } = require('@playwright/test');
+
+import AxeBuilder from '@axe-core/playwright';
+
+test.describe('Generic Webpage Tests', () => {
+ test('should load the webpage successfully', async ({ page }) => {
+ const response = await page.goto('/');
+ const title = await page.title();
+ await expect(response.status()).toBe(200);
+ });
+
+ test('should take a screenshot of the webpage', async ({ page }) => {
+ await page.goto('/');
+ await page.screenshot({ path: 'example-screenshot.png', fullPage: true });
+ });
+
+ // https://playwright.dev/docs/accessibility-testing
+ test('should not have any automatically detectable accessibility issues', async ({ page }) => {
+ await page.goto('/');
+ const accessibilityScanResults = await new AxeBuilder({ page }).analyze();
+ expect(accessibilityScanResults.violations).toEqual([]);
+ });
+
+ // Example test of finding a an html element on the index/home page
+ // test('should check for an element to be visible', async ({ page }) => {
+ // await page.goto('/');
+ // const element = page.locator('h1');
+ // await expect(element).toBeVisible();
+ // });
+
+});