From 65f672b3528a2c328dcabb01dd678516ac0bde3e Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 19 Apr 2024 18:33:50 +0200 Subject: [PATCH 01/20] Add initial playwright tests and fix discovered accessibility bugs --- .github/workflows/playwright.yml | 27 +++++++++++ .gitignore | 6 ++- README.md | 39 +++++++++++++++- package.json | 17 +++++++ playwright.config.ts | 78 ++++++++++++++++++++++++++++++++ pnpm-lock.yaml | 75 ++++++++++++++++++++++++++++++ templates/docs/with_menu.html | 4 +- templates/includes/header.html | 4 +- templates/index.html | 2 +- tests/menu.spec.ts | 26 +++++++++++ tests/metadata.spec.ts | 26 +++++++++++ 11 files changed, 297 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/playwright.yml create mode 100644 package.json create mode 100644 playwright.config.ts create mode 100644 pnpm-lock.yaml create mode 100644 tests/menu.spec.ts create mode 100644 tests/metadata.spec.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 000000000..9662b54e9 --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,27 @@ +name: Playwright Tests +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install dependencies + run: npm install -g pnpm && pnpm install + - name: Install Playwright Browsers + run: pnpm exec playwright install --with-deps + - name: Run Playwright tests + run: pnpm exec playwright test + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index 1d3301065..968a91b1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ /public .DS_Store gatsby - +node_modules/ +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/README.md b/README.md index 37c4e1f7d..e0dcd6b04 100644 --- a/README.md +++ b/README.md @@ -17,5 +17,42 @@ zola serve ``` Zola will build the website and start a web server, usually at -http://127.0.0.1:1111 + +## Tests + +We are using for simple UI and accessibility tests. + +## Running tests + +To run tests you need a few dependencies: + +- PNPM - The package manager used +- Playwright itself - Run `pnpm install` to install it +- The Plawright Browsers - Run `pnpm exec playwright install` to install those (The dependency errors are webkit specific and can be ignored) +- The system dependencies (This will not work on other systems than ubuntu or debian) - `sudo pnpm exec playwright install-deps` + +After installing the dependencies you can either run in headless mode using: + +``` +pnpm exec playwright test +``` + +Run in interactive UI mode using: + +``` +pnpm exec playwright test --ui +``` + +Or generate files using codegen: + +``` +pnpm exec playwright codegen +``` + +See the playwright documentation for more ways to launch tests. + +## Building tests + +For tests we are seeking mostly simple e2e tests ensuring things will work after changes. +Examples are things like our hover menu, accessibility or the rss feed. diff --git a/package.json b/package.json new file mode 100644 index 000000000..3f91e39a6 --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "matrix.org", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.43.1", + "@types/node": "^20.12.7" + }, + "dependencies": { + "@axe-core/playwright": "^4.9.0" + } +} diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 000000000..7041f6f96 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,78 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: 'http://127.0.0.1:1111', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + // Webkit dependencies are debian/ubuntu specific so it wont work on most other linux systems + process.env.CI ? { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + } : {}, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + webServer: { + command: 'zola serve', + url: 'http://127.0.0.1:1111', + reuseExistingServer: !process.env.CI, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 000000000..44a78bd5f --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,75 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + '@axe-core/playwright': + specifier: ^4.9.0 + version: 4.9.0(playwright-core@1.43.1) + +devDependencies: + '@playwright/test': + specifier: ^1.43.1 + version: 1.43.1 + '@types/node': + specifier: ^20.12.7 + version: 20.12.7 + +packages: + + /@axe-core/playwright@4.9.0(playwright-core@1.43.1): + resolution: {integrity: sha512-Q1Lz75dNsX38jof+aev7RficDMdH/HLOLySkDdXR0fUoeFcLdw4UNgDO2CNNP4CTpoesEdfYRdd6VmDXjhBgbA==} + peerDependencies: + playwright-core: '>= 1.0.0' + dependencies: + axe-core: 4.9.0 + playwright-core: 1.43.1 + dev: false + + /@playwright/test@1.43.1: + resolution: {integrity: sha512-HgtQzFgNEEo4TE22K/X7sYTYNqEMMTZmFS8kTq6m8hXj+m1D8TgwgIbumHddJa9h4yl4GkKb8/bgAl2+g7eDgA==} + engines: {node: '>=16'} + hasBin: true + dependencies: + playwright: 1.43.1 + dev: true + + /@types/node@20.12.7: + resolution: {integrity: sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==} + dependencies: + undici-types: 5.26.5 + dev: true + + /axe-core@4.9.0: + resolution: {integrity: sha512-H5orY+M2Fr56DWmMFpMrq5Ge93qjNdPVqzBv5gWK3aD1OvjBEJlEzxf09z93dGVQeI0LiW+aCMIx1QtShC/zUw==} + engines: {node: '>=4'} + dev: false + + /fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /playwright-core@1.43.1: + resolution: {integrity: sha512-EI36Mto2Vrx6VF7rm708qSnesVQKbxEWvPrfA1IPY6HgczBplDx7ENtx+K2n4kJ41sLLkuGfmb0ZLSSXlDhqPg==} + engines: {node: '>=16'} + hasBin: true + + /playwright@1.43.1: + resolution: {integrity: sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA==} + engines: {node: '>=16'} + hasBin: true + dependencies: + playwright-core: 1.43.1 + optionalDependencies: + fsevents: 2.3.2 + dev: true + + /undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + dev: true diff --git a/templates/docs/with_menu.html b/templates/docs/with_menu.html index 7eabf9713..9d48444e6 100644 --- a/templates/docs/with_menu.html +++ b/templates/docs/with_menu.html @@ -28,8 +28,8 @@ {% set current = false %} {% endif %}
- + diff --git a/templates/includes/header.html b/templates/includes/header.html index 8470aa708..3778d80de 100644 --- a/templates/includes/header.html +++ b/templates/includes/header.html @@ -11,8 +11,8 @@ {% set href = "/" ~ link.section ~ "/" %} {% set section = get_section(path=link.section ~ "/_index.md") %}
- + diff --git a/templates/index.html b/templates/index.html index 2b79ece4d..93834eec0 100644 --- a/templates/index.html +++ b/templates/index.html @@ -13,7 +13,7 @@
The Matrix Logo -

An open network for secure, decentralised communication

+

An open network for secure, decentralised communication

diff --git a/tests/menu.spec.ts b/tests/menu.spec.ts new file mode 100644 index 000000000..5ba194a84 --- /dev/null +++ b/tests/menu.spec.ts @@ -0,0 +1,26 @@ +import { test, expect } from '@playwright/test'; + +// We want to make sure that the hover has all children visible and is clickable +test('can select hovermenu', async ({ page }) => { + await page.goto('/'); + + await page.getByLabel('Ecosystem').hover(); + const elements = [ + page.locator(".section-submenu > a:nth-child(1)"), + page.locator(".section-submenu > a:nth-child(2)"), + page.locator(".section-submenu > a:nth-child(3)"), + page.locator(".section-submenu > a:nth-child(4)"), + page.locator(".section-submenu > a:nth-child(5)"), + page.locator(".section-submenu > a:nth-child(6)"), + ] + + await expect(page.locator(".section-submenu")).toBeVisible() + for (const element of elements) { + await expect(element).toBeVisible() + + // Ensure the elements itself can be hovered + await element.hover() + await expect(page.locator(".section-submenu")).toBeVisible() + await expect(element).toBeVisible() + } +}); diff --git a/tests/metadata.spec.ts b/tests/metadata.spec.ts new file mode 100644 index 000000000..84f7ca627 --- /dev/null +++ b/tests/metadata.spec.ts @@ -0,0 +1,26 @@ +import AxeBuilder from '@axe-core/playwright'; +import { test, expect } from '@playwright/test'; + +test('has title', async ({ page }) => { + await page.goto('/'); + + // Expect a title "to contain" Matrix.org + await expect(page).toHaveTitle(/Matrix.org/); + + // Ensure that the different subpages have their name in the title as well + await page.goto('blog') + + // Expect a title "to contain" Blog + await expect(page).toHaveTitle(/Blog/); +}); + + +test.describe('accessibility', () => { + test('homepage should not have any automatically detectable accessibility issues', async ({ page }) => { + await page.goto('/'); + + const accessibilityScanResults = await new AxeBuilder({ page }).analyze(); + + expect(accessibilityScanResults.violations).toEqual([]); + }); +}); From a28a9e47d44f93ebcb1a5ef664bafc3e559042de Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 19 Apr 2024 18:35:30 +0200 Subject: [PATCH 02/20] Ensure we run sharded tests on CI --- .github/workflows/playwright.yml | 54 ++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 9662b54e9..e9401821b 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -1,13 +1,18 @@ name: Playwright Tests on: push: - branches: [ main, master ] + branches: [ main ] pull_request: - branches: [ main, master ] + branches: [ main ] jobs: - test: + playwright-tests: timeout-minutes: 60 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4] + shardTotal: [4] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -18,10 +23,41 @@ jobs: - name: Install Playwright Browsers run: pnpm exec playwright install --with-deps - name: Run Playwright tests - run: pnpm exec playwright test - - uses: actions/upload-artifact@v4 - if: always() + run: pnpm exec playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + - name: Upload blob report to GitHub Actions Artifacts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 with: - name: playwright-report - path: playwright-report/ - retention-days: 30 + name: blob-report-${{ matrix.shardIndex }} + path: blob-report + retention-days: 1 + merge-reports: + # Merge reports after playwright-tests, even if some shards have failed + if: ${{ !cancelled() }} + needs: [playwright-tests] + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Install dependencies + run: npm ci + + - name: Download blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@v4 + with: + path: all-blob-reports + pattern: blob-report-* + merge-multiple: true + + - name: Merge into HTML Report + run: npx playwright merge-reports --reporter html ./all-blob-reports + + - name: Upload HTML report + uses: actions/upload-artifact@v4 + with: + name: html-report--attempt-${{ github.run_attempt }} + path: playwright-report + retention-days: 14 From 90bbd8941b158d06497e9caa5fd0b9017d0978ee Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 19 Apr 2024 18:38:07 +0200 Subject: [PATCH 03/20] Ensure zola is installed --- .github/workflows/playwright.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index e9401821b..4afbae1bc 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -15,6 +15,12 @@ jobs: shardTotal: [4] steps: - uses: actions/checkout@v4 + - name: Download Zola + run: | + version='v0.17.2' + archive="zola-${version}-x86_64-unknown-linux-gnu.tar.gz" + curl -L "https://github.com/getzola/zola/releases/download/${version}/${archive}" -o "$archive" + tar xf "$archive" - uses: actions/setup-node@v4 with: node-version: lts/* From a8de533889d80c0c04c8ee1e9db2cddafe0e5be6 Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 19 Apr 2024 18:41:49 +0200 Subject: [PATCH 04/20] Ensure zola is is in path --- .github/workflows/playwright.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 4afbae1bc..d8967af8f 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -21,6 +21,7 @@ jobs: archive="zola-${version}-x86_64-unknown-linux-gnu.tar.gz" curl -L "https://github.com/getzola/zola/releases/download/${version}/${archive}" -o "$archive" tar xf "$archive" + pwd >> $GITHUB_PATH - uses: actions/setup-node@v4 with: node-version: lts/* From 05daa1d27406f7b6f1526de9973b2cbe3eb72872 Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 19 Apr 2024 18:43:57 +0200 Subject: [PATCH 05/20] Use blob reporter in CI --- playwright.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playwright.config.ts b/playwright.config.ts index 7041f6f96..16c4c8f09 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -20,7 +20,7 @@ export default defineConfig({ /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', + reporter: process.env.CI ? 'blob' : 'html', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ From bf482747cbb1f9ccd2c4cbc67e7762329c36ea9b Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 19 Apr 2024 18:44:53 +0200 Subject: [PATCH 06/20] Use correct commands when merging the reports from playwright --- .github/workflows/playwright.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index d8967af8f..35637cf6b 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -50,7 +50,7 @@ jobs: with: node-version: 18 - name: Install dependencies - run: npm ci + run: npm install -g pnpm && pnpm install - name: Download blob reports from GitHub Actions Artifacts uses: actions/download-artifact@v4 @@ -60,7 +60,7 @@ jobs: merge-multiple: true - name: Merge into HTML Report - run: npx playwright merge-reports --reporter html ./all-blob-reports + run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports - name: Upload HTML report uses: actions/upload-artifact@v4 From 0160c5f5e9cbea07b3129cb6168639196c88555d Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 19 Apr 2024 18:51:04 +0200 Subject: [PATCH 07/20] Fix the lost css on the hero of the frontpage --- sass/_index.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sass/_index.scss b/sass/_index.scss index 6229e05b4..ecb81a2bb 100644 --- a/sass/_index.scss +++ b/sass/_index.scss @@ -24,7 +24,7 @@ flex-direction: column; justify-content: center; - h2 { + h1 { color: #fff; font-size: 2.5rem; font-weight: 700; From 07bb4fceb2138d49778636f948c3bac0d7f49bab Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 19 Apr 2024 19:09:28 +0200 Subject: [PATCH 08/20] Add accessibility tests for the blog page and add a missing aria-label for the pagination --- templates/includes/pagination.html | 2 +- tests/metadata.spec.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/templates/includes/pagination.html b/templates/includes/pagination.html index 0a63d0544..7f5c5e60c 100644 --- a/templates/includes/pagination.html +++ b/templates/includes/pagination.html @@ -1,4 +1,4 @@ -