Skip to content

Commit

Permalink
Switch to TestContainers for manging services in Playwright (#28860)
Browse files Browse the repository at this point in the history
* Switch to TestContainers for manging services in Playwright

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Flip fixture dependency order

Signed-off-by: Michael Telatynski <[email protected]>

* Remove mas dep

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Update matrix-authentication-service in Playwright tests

Signed-off-by: Michael Telatynski <[email protected]>

* delint

Signed-off-by: Michael Telatynski <[email protected]>

* Fix SMTP port

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Comments

Signed-off-by: Michael Telatynski <[email protected]>

* Strip ansi from playwright logs to make them more readable

Signed-off-by: Michael Telatynski <[email protected]>

* Actually do the update

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Remove access to homeserver.config.baseUrl field in favour of homeserver.baseUrl

Signed-off-by: Michael Telatynski <[email protected]>

* Use sane default_server_config and specify server.invalid in the specific tests which demand it

Signed-off-by: Michael Telatynski <[email protected]>

* Fix mas run

Signed-off-by: Michael Telatynski <[email protected]>

* break cycle

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* typo

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* Iterate

Signed-off-by: Michael Telatynski <[email protected]>

* prettier

Signed-off-by: Michael Telatynski <[email protected]>

* Wire up basics of dendriteHomeserver

Signed-off-by: Michael Telatynski <[email protected]>

---------

Signed-off-by: Michael Telatynski <[email protected]>
  • Loading branch information
t3chguy authored Jan 7, 2025
1 parent 66bbb84 commit f75d1f5
Show file tree
Hide file tree
Showing 61 changed files with 1,925 additions and 2,706 deletions.
6 changes: 5 additions & 1 deletion docs/playwright.md
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,11 @@ has to be disabled in Playwright on Firefox & Webkit to retain routing functiona
Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available
there at this time.

## Colima
## Supporter container runtimes

We use testcontainers to spin up various instances of Synapse, Matrix Authentication Service, and more.
It supports Docker out of the box but also has support for Podman, Colima, Rancher, you just need to follow some instructions to achieve it:
https://node.testcontainers.org/supported-container-runtimes/

If you are running under Colima, you may need to set the environment variable `TMPDIR` to `/tmp/colima` or a path
within `$HOME` to allow bind mounting temporary directories into the Docker containers.
5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@
"@sentry/webpack-plugin": "^2.7.1",
"@stylistic/eslint-plugin": "^2.9.0",
"@svgr/webpack": "^8.0.0",
"@testcontainers/postgresql": "^10.16.0",
"@testing-library/dom": "^10.4.0",
"@testing-library/jest-dom": "^6.4.8",
"@testing-library/react": "^16.0.0",
Expand All @@ -191,7 +192,6 @@
"@types/escape-html": "^1.0.1",
"@types/express": "^5.0.0",
"@types/file-saver": "^2.0.3",
"@types/fs-extra": "^11.0.0",
"@types/glob-to-regexp": "^0.4.1",
"@types/jest": "29.5.12",
"@types/jitsi-meet": "^2.0.2",
Expand Down Expand Up @@ -243,7 +243,6 @@
"fetch-mock": "9.11.0",
"fetch-mock-jest": "^1.5.1",
"file-loader": "^6.0.0",
"fs-extra": "^11.0.0",
"glob": "^11.0.0",
"html-webpack-plugin": "^5.5.3",
"husky": "^9.0.0",
Expand Down Expand Up @@ -278,11 +277,13 @@
"rimraf": "^6.0.0",
"semver": "^7.5.2",
"source-map-loader": "^5.0.0",
"strip-ansi": "^7.1.0",
"stylelint": "^16.1.0",
"stylelint-config-standard": "^36.0.0",
"stylelint-scss": "^6.0.0",
"stylelint-value-no-unknown-custom-properties": "^6.0.1",
"terser-webpack-plugin": "^5.3.9",
"testcontainers": "^10.16.0",
"ts-node": "^10.9.1",
"ts-prune": "^0.10.3",
"typescript": "5.7.2",
Expand Down
4 changes: 3 additions & 1 deletion playwright/e2e/app-loading/guest-registration.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ Please see LICENSE files in the repository root for full details.
import { expect, test } from "../../element-web-test";

test.use({
startHomeserverOpts: "guest-enabled",
synapseConfigOptions: {
allow_guest_access: true,
},
});

test("Shows the welcome page by default", async ({ page }) => {
Expand Down
88 changes: 43 additions & 45 deletions playwright/e2e/crypto/backups.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ Please see LICENSE files in the repository root for full details.
import { type Page } from "@playwright/test";

import { test, expect } from "../../element-web-test";
import { test as masTest, registerAccountMas } from "../oidc";
import { registerAccountMas } from "../oidc";
import { isDendrite } from "../../plugins/homeserver/dendrite";
import { TestClientServerAPI } from "../csAPI";
import { masHomeserver } from "../../plugins/homeserver/synapse/masHomeserver.ts";

async function expectBackupVersionToBe(page: Page, version: string) {
await expect(page.locator(".mx_SecureBackupPanel_statusList tr:nth-child(5) td")).toHaveText(
Expand All @@ -24,22 +25,23 @@ async function expectBackupVersionToBe(page: Page, version: string) {
// These tests register an account with MAS because then we go through the "normal" registration flow
// and crypto gets set up. Using the 'user' fixture create a a user an synthesizes an existing login,
// which is faster but leaves us without crypto set up.
masTest.describe("Encryption state after registration", () => {
masTest.skip(isDendrite, "does not yet support MAS");
test.describe("Encryption state after registration", () => {
test.use(masHomeserver);
test.skip(isDendrite, "does not yet support MAS");

masTest("Key backup is enabled by default", async ({ page, mailhog, app }) => {
test("Key backup is enabled by default", async ({ page, mailhogClient, app }) => {
await page.goto("/#/login");
await page.getByRole("button", { name: "Continue" }).click();
await registerAccountMas(page, mailhog.api, "alice", "[email protected]", "Pa$sW0rD!");
await registerAccountMas(page, mailhogClient, "alice", "[email protected]", "Pa$sW0rD!");

await app.settings.openUserSettings("Security & Privacy");
await expect(page.getByText("This session is backing up your keys.")).toBeVisible();
});

masTest("user is prompted to set up recovery", async ({ page, mailhog, app }) => {
test("user is prompted to set up recovery", async ({ page, mailhogClient, app }) => {
await page.goto("/#/login");
await page.getByRole("button", { name: "Continue" }).click();
await registerAccountMas(page, mailhog.api, "alice", "[email protected]", "Pa$sW0rD!");
await registerAccountMas(page, mailhogClient, "alice", "[email protected]", "Pa$sW0rD!");

await page.getByRole("button", { name: "Add room" }).click();
await page.getByRole("menuitem", { name: "New room" }).click();
Expand All @@ -50,57 +52,53 @@ masTest.describe("Encryption state after registration", () => {
});
});

masTest.describe("Key backup reset from elsewhere", () => {
masTest.skip(isDendrite, "does not yet support MAS");
test.describe("Key backup reset from elsewhere", () => {
test.use(masHomeserver);
test.skip(isDendrite, "does not yet support MAS");

masTest(
"Key backup is disabled when reset from elsewhere",
async ({ page, mailhog, request, masPrepare, homeserver }) => {
const testUsername = "alice";
const testPassword = "Pa$sW0rD!";
test("Key backup is disabled when reset from elsewhere", async ({ page, mailhogClient, request, homeserver }) => {
const testUsername = "alice";
const testPassword = "Pa$sW0rD!";

// there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake
// clock so we can skip the delay
await page.clock.install();
// there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake
// clock so we can skip the delay
await page.clock.install();

await page.goto("/#/login");
await page.getByRole("button", { name: "Continue" }).click();
await registerAccountMas(page, mailhog.api, testUsername, "[email protected]", testPassword);
await page.goto("/#/login");
await page.getByRole("button", { name: "Continue" }).click();
await registerAccountMas(page, mailhogClient, testUsername, "[email protected]", testPassword);

await page.getByRole("button", { name: "Add room" }).click();
await page.getByRole("menuitem", { name: "New room" }).click();
await page.getByRole("textbox", { name: "Name" }).fill("test room");
await page.getByRole("button", { name: "Create room" }).click();
await page.getByRole("button", { name: "Add room" }).click();
await page.getByRole("menuitem", { name: "New room" }).click();
await page.getByRole("textbox", { name: "Name" }).fill("test room");
await page.getByRole("button", { name: "Create room" }).click();

// @ts-ignore - this runs in the browser scope where mxMatrixClientPeg is a thing. Here, it is not.
const accessToken = await page.evaluate(() => mxMatrixClientPeg.get().getAccessToken());
// @ts-ignore - this runs in the browser scope where mxMatrixClientPeg is a thing. Here, it is not.
const accessToken = await page.evaluate(() => mxMatrixClientPeg.get().getAccessToken());

const csAPI = new TestClientServerAPI(request, homeserver, accessToken);
const csAPI = new TestClientServerAPI(request, homeserver, accessToken);

const backupInfo = await csAPI.getCurrentBackupInfo();
const backupInfo = await csAPI.getCurrentBackupInfo();

await csAPI.deleteBackupVersion(backupInfo.version);
await csAPI.deleteBackupVersion(backupInfo.version);

await page.getByRole("textbox", { name: "Send an encrypted message…" }).fill("/discardsession");
await page.getByRole("button", { name: "Send message" }).click();
await page.getByRole("textbox", { name: "Send an encrypted message…" }).fill("/discardsession");
await page.getByRole("button", { name: "Send message" }).click();

await page
.getByRole("textbox", { name: "Send an encrypted message…" })
.fill("Message with broken key backup");
await page.getByRole("button", { name: "Send message" }).click();
await page.getByRole("textbox", { name: "Send an encrypted message…" }).fill("Message with broken key backup");
await page.getByRole("button", { name: "Send message" }).click();

// Should be the message we sent plus the room creation event
await expect(page.locator(".mx_EventTile")).toHaveCount(2);
await expect(
page.locator(".mx_RoomView_MessageList > .mx_EventTile_last .mx_EventTile_receiptSent"),
).toBeVisible();
// Should be the message we sent plus the room creation event
await expect(page.locator(".mx_EventTile")).toHaveCount(2);
await expect(
page.locator(".mx_RoomView_MessageList > .mx_EventTile_last .mx_EventTile_receiptSent"),
).toBeVisible();

// Wait for it to try uploading the key
await page.clock.fastForward(20000);
// Wait for it to try uploading the key
await page.clock.fastForward(20000);

await expect(page.getByRole("heading", { level: 1, name: "New Recovery Method" })).toBeVisible();
},
);
await expect(page.getByRole("heading", { level: 1, name: "New Recovery Method" })).toBeVisible();
});
});

test.describe("Backups", () => {
Expand Down
39 changes: 19 additions & 20 deletions playwright/e2e/crypto/dehydration.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,10 @@ Please see LICENSE files in the repository root for full details.

import { Locator, type Page } from "@playwright/test";

import { test as base, expect, Fixtures } from "../../element-web-test";
import { test, expect } from "../../element-web-test";
import { viewRoomSummaryByName } from "../right-panel/utils";
import { isDendrite } from "../../plugins/homeserver/dendrite";

const test = base.extend<Fixtures>({
// eslint-disable-next-line no-empty-pattern
startHomeserverOpts: async ({}, use) => {
await use("dehydration");
},
config: async ({ config, context }, use) => {
const wellKnown = {
...config.default_server_config,
"org.matrix.msc3814": true,
};

await context.route("https://localhost/.well-known/matrix/client", async (route) => {
await route.fulfill({ json: wellKnown });
});

await use(config);
},
});

const ROOM_NAME = "Test room";
const NAME = "Alice";

Expand All @@ -43,6 +24,24 @@ test.describe("Dehydration", () => {

test.use({
displayName: NAME,
synapseConfigOptions: {
experimental_features: {
msc2697_enabled: false,
msc3814_enabled: true,
},
},
config: async ({ config, context }, use) => {
const wellKnown = {
...config.default_server_config,
"org.matrix.msc3814": true,
};

await context.route("https://localhost/.well-known/matrix/client", async (route) => {
await route.fulfill({ json: wellKnown });
});

await use(config);
},
});

test("Create dehydrated device", async ({ page, user, app }, workerInfo) => {
Expand Down
32 changes: 16 additions & 16 deletions playwright/e2e/crypto/migration.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,24 @@ Please see LICENSE files in the repository root for full details.
import path from "path";
import { readFile } from "node:fs/promises";

import { expect, Fixtures, test as base } from "../../element-web-test";

const test = base.extend<Fixtures>({
// Replace the `user` fixture with one which populates the indexeddb data before starting the app.
user: async ({ context, pageWithCredentials: page, credentials }, use) => {
await page.route(`/test_indexeddb_cryptostore_dump/*`, async (route, request) => {
const resourcePath = path.join(__dirname, new URL(request.url()).pathname);
const body = await readFile(resourcePath, { encoding: "utf-8" });
await route.fulfill({ body });
});
await page.goto("/test_indexeddb_cryptostore_dump/index.html");

await use(credentials);
},
});
import { expect, test } from "../../element-web-test";

test.describe("migration", { tag: "@no-webkit" }, function () {
test.use({ displayName: "Alice" });
test.use({
displayName: "Alice",

// Replace the `user` fixture with one which populates the indexeddb data before starting the app.
user: async ({ context, pageWithCredentials: page, credentials }, use) => {
await page.route(`/test_indexeddb_cryptostore_dump/*`, async (route, request) => {
const resourcePath = path.join(__dirname, new URL(request.url()).pathname);
const body = await readFile(resourcePath, { encoding: "utf-8" });
await route.fulfill({ body });
});
await page.goto("/test_indexeddb_cryptostore_dump/index.html");

await use(credentials);
},
});

test("Should support migration from legacy crypto", async ({ context, user, page }, workerInfo) => {
test.slow();
Expand Down
12 changes: 4 additions & 8 deletions playwright/e2e/forgot-password/forgot-password.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,17 @@ Please see LICENSE files in the repository root for full details.

import { expect, test } from "../../element-web-test";
import { selectHomeserver } from "../utils";
import { emailHomeserver } from "../../plugins/homeserver/synapse/emailHomeserver.ts";
import { isDendrite } from "../../plugins/homeserver/dendrite";

const username = "user1234";
// this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen.
const password = "oETo7MPf0o";
const email = "[email protected]";

test.describe("Forgot Password", () => {
test.skip(isDendrite, "not yet wired up");
test.use(emailHomeserver);
test.use({
config: {
// The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
Expand All @@ -25,14 +29,6 @@ test.describe("Forgot Password", () => {
},
},
},
startHomeserverOpts: ({ mailhog }, use) =>
use({
template: "email",
variables: {
SMTP_HOST: "host.containers.internal",
SMTP_PORT: mailhog.instance.smtpPort,
},
}),
});

test("renders properly", { tag: "@screenshot" }, async ({ page, homeserver }) => {
Expand Down
3 changes: 2 additions & 1 deletion playwright/e2e/login/consent.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ Please see LICENSE files in the repository root for full details.
*/

import { test, expect } from "../../element-web-test";
import { consentHomeserver } from "../../plugins/homeserver/synapse/consentHomeserver.ts";

test.describe("Consent", () => {
test.use(consentHomeserver);
test.use({
startHomeserverOpts: "consent",
displayName: "Bob",
});

Expand Down
17 changes: 6 additions & 11 deletions playwright/e2e/login/login.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import { doTokenRegistration } from "./utils";
import { isDendrite } from "../../plugins/homeserver/dendrite";
import { selectHomeserver } from "../utils";
import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
import { consentHomeserver } from "../../plugins/homeserver/synapse/consentHomeserver.ts";
import { legacyOAuthHomeserver } from "../../plugins/homeserver/synapse/legacyOAuthHomeserver.ts";

const username = "user1234";
const password = "p4s5W0rD";
Expand Down Expand Up @@ -91,7 +93,7 @@ test.describe("Login", () => {
});

test.describe("Password login", () => {
test.use({ startHomeserverOpts: "consent" });
test.use(consentHomeserver);

let creds: Credentials;

Expand Down Expand Up @@ -238,14 +240,7 @@ test.describe("Login", () => {
// tests for old-style SSO login, in which we exchange tokens with Synapse, and Synapse talks to an auth server
test.describe("SSO login", () => {
test.skip(isDendrite, "does not yet support SSO");

test.use({
startHomeserverOpts: ({ oAuthServer }, use) =>
use({
template: "default",
oAuthServerPort: oAuthServer.port,
}),
});
test.use(legacyOAuthHomeserver);

test("logs in with SSO and lands on the home screen", async ({ page, homeserver }) => {
// If this test fails with a screen showing "Timeout connecting to remote server", it is most likely due to
Expand All @@ -259,7 +254,7 @@ test.describe("Login", () => {
});

test.describe("logout", () => {
test.use({ startHomeserverOpts: "consent" });
test.use(consentHomeserver);

test("should go to login page on logout", async ({ page, user }) => {
await page.getByRole("button", { name: "User menu" }).click();
Expand All @@ -274,8 +269,8 @@ test.describe("Login", () => {
});

test.describe("logout with logout_redirect_url", () => {
test.use(consentHomeserver);
test.use({
startHomeserverOpts: "consent",
config: {
// We redirect to decoder-ring because it's a predictable page that isn't Element itself.
// We could use example.org, matrix.org, or something else, however this puts dependency of external
Expand Down
Loading

0 comments on commit f75d1f5

Please sign in to comment.