Testing

.github/workflows/playwright.yml

@@ -0,0 +1,70 @@
+name: Playwright Tests
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+ branches: [ main ]
+jobs:
+ playwright-tests:
+ timeout-minutes: 60
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ shardIndex: [1, 2, 3, 4]
+ shardTotal: [4]
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download Zola
+ run: |
+ version='v0.17.2'
+ archive="zola-${version}-x86_64-unknown-linux-gnu.tar.gz"
+ curl -L "https://github.com/getzola/zola/releases/download/${version}/${archive}" -o "$archive"
+ tar xf "$archive"
+ pwd >> $GITHUB_PATH
+ - uses: actions/setup-node@v4
+ with:
+ node-version: lts/*
+ - name: Install dependencies
+ run: npm install -g pnpm && pnpm install
+ - name: Install Playwright Browsers
+ run: pnpm exec playwright install --with-deps
+ - name: Run Playwright tests
+ run: pnpm exec playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+ - name: Upload blob report to GitHub Actions Artifacts
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: blob-report-${{ matrix.shardIndex }}
+ path: blob-report
+ retention-days: 1
+ merge-reports:
+ # Merge reports after playwright-tests, even if some shards have failed
+ if: ${{ !cancelled() }}
+ needs: [playwright-tests]
+
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 18
+ - name: Install dependencies
+ run: npm install -g pnpm && pnpm install
+
+ - name: Download blob reports from GitHub Actions Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ path: all-blob-reports
+ pattern: blob-report-*
+ merge-multiple: true
+
+ - name: Merge into HTML Report
+ run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports
+
+ - name: Upload HTML report
+ uses: actions/upload-artifact@v4
+ with:
+ name: html-report--attempt-${{ github.run_attempt }}
+ path: playwright-report
+ retention-days: 14

.gitignore

@@ -1,4 +1,8 @@
 /public
 .DS_Store
 gatsby
-
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

README.md

@@ -17,5 +17,42 @@ zola serve
 ```
 
 Zola will build the website and start a web server, usually at
-http://127.0.0.1:1111
+<http://127.0.0.1:1111>
 
+## Tests
+
+We are using <https://playwright.dev/> for simple UI and accessibility tests.
+
+## Running tests
+
+To run tests you need a few dependencies:
+
+- PNPM - The package manager used
+- Playwright itself - Run `pnpm install` to install it
+- The Plawright Browsers - Run `pnpm exec playwright install` to install those (The dependency errors are webkit specific and can be ignored)
+- The system dependencies (This will not work on other systems than ubuntu or debian) - `sudo pnpm exec playwright install-deps`
+
+After installing the dependencies you can either run in headless mode using:
+
+```
+pnpm exec playwright test
+```
+
+Run in interactive UI mode using:
+
+```
+pnpm exec playwright test --ui
+```
+
+Or generate files using codegen:
+
+```
+pnpm exec playwright codegen
+```
+
+See the playwright documentation for more ways to launch tests.
+
+## Building tests
+
+For tests we are seeking mostly simple e2e tests ensuring things will work after changes.
+Examples are things like our hover menu, accessibility or the rss feed.

package.json

@@ -0,0 +1,19 @@
+{
+ "name": "matrix.org",
+ "version": "1.0.0",
+ "private": true,
+ "main": "index.js",
+ "scripts": {
+ "test": "playwright test",
+ "serve": "zola serve"
+ },
+ "author": "The Matrix.org Foundation and Contributors",
+ "license": "Apache-2.0",
+ "devDependencies": {
+ "@playwright/test": "^1.43.1",
+ "@types/node": "^20.12.7"
+ },
+ "dependencies": {
+ "@axe-core/playwright": "^4.9.0"
+ }
+}

playwright.config.ts

@@ -0,0 +1,78 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+ testDir: './tests',
+ /* Run tests in files in parallel */
+ fullyParallel: true,
+ /* Fail the build on CI if you accidentally left test.only in the source code. */
+ forbidOnly: !!process.env.CI,
+ /* Retry on CI only */
+ retries: process.env.CI ? 2 : 0,
+ /* Opt out of parallel tests on CI. */
+ workers: process.env.CI ? 1 : undefined,
+ /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+ reporter: process.env.CI ? 'blob' : 'html',
+ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+ use: {
+ /* Base URL to use in actions like `await page.goto('/')`. */
+ baseURL: 'http://127.0.0.1:1111',
+
+ /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+ trace: 'on-first-retry',
+ },
+
+ /* Configure projects for major browsers */
+ projects: [
+ {
+ name: 'chromium',
+ use: { ...devices['Desktop Chrome'] },
+ },
+
+ {
+ name: 'firefox',
+ use: { ...devices['Desktop Firefox'] },
+ },
+
+ // Webkit dependencies are debian/ubuntu specific so it wont work on most other linux systems
+ process.env.CI ? {
+ name: 'webkit',
+ use: { ...devices['Desktop Safari'] },
+ } : {},
+
+ /* Test against mobile viewports. */
+ // {
+ // name: 'Mobile Chrome',
+ // use: { ...devices['Pixel 5'] },
+ // },
+ // {
+ // name: 'Mobile Safari',
+ // use: { ...devices['iPhone 12'] },
+ // },
+
+ /* Test against branded browsers. */
+ // {
+ // name: 'Microsoft Edge',
+ // use: { ...devices['Desktop Edge'], channel: 'msedge' },
+ // },
+ // {
+ // name: 'Google Chrome',
+ // use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+ // },
+ ],
+
+ /* Run your local dev server before starting the tests */
+ webServer: {
+ command: 'zola serve',
+ url: 'http://127.0.0.1:1111',
+ reuseExistingServer: !process.env.CI,
+ },
+});

sass/_clients.scss

@@ -73,7 +73,7 @@ body:has(.projects-card-deck > div > .client-checkbox:checked) {
  margin-inline: auto;
  }
 
- h3 {
+ h1 {
  display: flex;
  gap: .5rem;
  margin: 0 auto;

sass/_index.scss

@@ -24,7 +24,7 @@
  flex-direction: column;
  justify-content: center;
 
- h2 {
+ h1 {
  color: #fff;
  font-size: 2.5rem;
  font-weight: 700;

sass/_try-matrix.scss

@@ -31,7 +31,7 @@
 
  padding-block: 1rem;
 
- h3 {
+ h2 {
  font-size: 2.2rem;
  margin-top: 0;
  }

templates/docs/with_menu.html

@@ -28,8 +28,8 @@
  {% set current = false %}
  {% endif %}
  <div>
- <input id="{{ doc_section.title }}-submenu-checkbox" type="checkbox" class="submenu-checkbox"
- aria-hidden="true" {% if current %}checked{% endif %}>
+ <input id="{{ doc_section.title }}-submenu-checkbox" type="checkbox" class="submenu-checkbox" {% if
+ current %}checked{% endif %}>
  <label for="{{ doc_section.title }}-submenu-checkbox" class="submenu-title">{{ doc_section.title }}
  <div class="arrow"></div>
  </label>

templates/ecosystem/client.html

@@ -14,9 +14,9 @@
  <div class="details">
  <img class="app-logo" src="../{{ thumbnail }}" alt="{{ page.title }} Logo">
  <div class="name-and-release-date">
- <h3>{{ page.title }}<div class="maturity {{ page.extra.maturity | lower }}">{{ page.extra.maturity }}
+ <h1>{{ page.title }}<div class="maturity {{ page.extra.maturity | lower }}">{{ page.extra.maturity }}
  </div>
- </h3>
+ </h1>
  </div>
  <p class="summary">{{ page.content | markdown | safe }}</p>
  <div class="project-links">

templates/includes/header.html

@@ -11,8 +11,8 @@
  {% set href = "/" ~ link.section ~ "/" %}
  {% set section = get_section(path=link.section ~ "/_index.md") %}
  <div class="section-wrap">
- <input id="{{ link.section }}-submenu-checkbox" type="checkbox" class="submenu-checkbox" aria-hidden="true"
- {{ util::current_checked(if_path_starts_with="/" ~ link.section) }}>
+ <input id="{{ link.section }}-submenu-checkbox" type="checkbox" class="submenu-checkbox" {{
+ util::current_checked(if_path_starts_with="/" ~ link.section) }}>
  <label for="{{ link.section }}-submenu-checkbox" class="submenu-title">{{ link.title }} <div class="arrow">
  </div></label>
 

templates/includes/pagination.html

@@ -1,4 +1,4 @@
-<nav class="pagination">
+<nav class="pagination" aria-label="Blog Pagination Section">
  <div class="prev">
  {% if paginator.previous %}
  <a href="{{ paginator.previous }}">‹ Previous</a>

templates/index.html

@@ -13,7 +13,7 @@
  <div id="hero" class="section">
  <div class="hero_content">
  <img src="images/matrix-logo-white.svg" alt="The Matrix Logo" />
- <h2>An open network for secure, decentralised communication</h2>
+ <h1>An open network for secure, decentralised communication</h1>
  </div>
  <div id="hero_curve1"></div>
  <div id="hero_curve2"></div>

templates/macros/sdks.html

@@ -85,7 +85,7 @@
  <div
  class="project-card maturity-{{ sdk.maturity | lower }} licence-{{ sdk.licence | slugify }} language-{{ sdk.language | slugify }}">
  <div class="title-row">
- <h3>{{ sdk.name }}</h3>
+ <h2>{{ sdk.name }}</h2>
  <div class="maturity {{ sdk.maturity | lower }}">{{ sdk.maturity }}</div>
  </div>
  <div class="author">{{ sdk.maintainer }}</div>

templates/shortcodes/all_integrations.html

@@ -2,7 +2,7 @@
 <div class="projects-card-deck">
  {% for integration in integrations_data.integrations %}
  <div class="project-card">
- <h3>{{ integration.name }}</h3>
+ <h2>{{ integration.name }}</h2>
  <span>{{ integration.description | safe }}</span>
  <div class="server-links">
  {% if integration.repository %}