Fix flaky playwright tests

docs/playwright.md

@@ -77,6 +77,9 @@ test.use({
 ```
 
 The appropriate homeserver will be launched by the Playwright worker and reused for all tests which match the worker configuration.
+Due to homeservers being reused between tests, please use unique names for any rooms put into the room directory as
+they may be visible from other tests, the suggested approach is to use `testInfo.testId` within the name or lodash's uniqueId.
+We remove public rooms from the directory between tests but deleting users doesn't have a homeserver agnostic solution.
 The logs from testcontainers will be attached to any reports output from Playwright.
 
 ## Writing Tests

playwright/e2e/crypto/backups-mas.spec.ts

@@ -19,19 +19,19 @@ test.use(masHomeserver);
 test.describe("Encryption state after registration", () => {
     test.skip(isDendrite, "does not yet support MAS");
 
-    test("Key backup is enabled by default", async ({ page, mailhogClient, app }) => {
+    test("Key backup is enabled by default", async ({ page, mailhogClient, app }, testInfo) => {
         await page.goto("/#/login");
         await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "[email protected]", "Pa$sW0rD!");
+        await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "[email protected]", "Pa$sW0rD!");
 
         await app.settings.openUserSettings("Security & Privacy");
         await expect(page.getByText("This session is backing up your keys.")).toBeVisible();
     });
 
-    test("user is prompted to set up recovery", async ({ page, mailhogClient, app }) => {
+    test("user is prompted to set up recovery", async ({ page, mailhogClient, app }, testInfo) => {
         await page.goto("/#/login");
         await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "[email protected]", "Pa$sW0rD!");
+        await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "[email protected]", "Pa$sW0rD!");
 
         await page.getByRole("button", { name: "Add room" }).click();
         await page.getByRole("menuitem", { name: "New room" }).click();
@@ -45,8 +45,13 @@ test.describe("Encryption state after registration", () => {
 test.describe("Key backup reset from elsewhere", () => {
     test.skip(isDendrite, "does not yet support MAS");
 
-    test("Key backup is disabled when reset from elsewhere", async ({ page, mailhogClient, request, homeserver }) => {
-        const testUsername = "alice";
+    test("Key backup is disabled when reset from elsewhere", async ({
+        page,
+        mailhogClient,
+        request,
+        homeserver,
+    }, testInfo) => {
+        const testUsername = `alice_${testInfo.testId}`;
         const testPassword = "Pa$sW0rD!";
 
         // there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake
@@ -62,8 +67,7 @@ test.describe("Key backup reset from elsewhere", () => {
         await page.getByRole("textbox", { name: "Name" }).fill("test room");
         await page.getByRole("button", { name: "Create room" }).click();
 
-        // @ts-ignore - this runs in the browser scope where mxMatrixClientPeg is a thing. Here, it is not.
-        const accessToken = await page.evaluate(() => mxMatrixClientPeg.get().getAccessToken());
+        const accessToken = await page.evaluate(() => window.mxMatrixClientPeg.get().getAccessToken());
 
         const csAPI = new TestClientServerAPI(request, homeserver, accessToken);
 

playwright/e2e/csAPI.ts

@@ -9,40 +9,31 @@ import { APIRequestContext } from "playwright-core";
 import { KeyBackupInfo } from "matrix-js-sdk/src/crypto-api";
 
 import { HomeserverInstance } from "../plugins/homeserver";
+import { ClientServerApi } from "../testcontainers/utils.ts";
 
 /**
  * A small subset of the Client-Server API used to manipulate the state of the
  * account on the homeserver independently of the client under test.
  */
-export class TestClientServerAPI {
+export class TestClientServerAPI extends ClientServerApi {
     public constructor(
-        private request: APIRequestContext,
-        private homeserver: HomeserverInstance,
+        request: APIRequestContext,
+        homeserver: HomeserverInstance,
         private accessToken: string,
-    ) {}
+    ) {
+        super(homeserver.baseUrl);
+        this.setRequest(request);
+    }
 
     public async getCurrentBackupInfo(): Promise<KeyBackupInfo | null> {
-        const res = await this.request.get(`${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version`, {
-            headers: { Authorization: `Bearer ${this.accessToken}` },
-        });
-
-        return await res.json();
+        return this.request("GET", `/v3/room_keys/version`, this.accessToken);
     }
 
     /**
      * Calls the API directly to delete the given backup version
      * @param version The version to delete
      */
     public async deleteBackupVersion(version: string): Promise<void> {
-        const res = await this.request.delete(
-            `${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version/${version}`,
-            {
-                headers: { Authorization: `Bearer ${this.accessToken}` },
-            },
-        );
-
-        if (!res.ok) {
-            throw new Error(`Failed to delete backup version: ${res.status}`);
-        }
+        await this.request("DELETE", `/v3/room_keys/version/${version}`, this.accessToken);
     }
 }

playwright/e2e/forgot-password/forgot-password.spec.ts

@@ -6,16 +6,25 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { expect, test } from "../../element-web-test";
+import { expect, test as base } from "../../element-web-test";
 import { selectHomeserver } from "../utils";
 import { emailHomeserver } from "../../plugins/homeserver/synapse/emailHomeserver.ts";
 import { isDendrite } from "../../plugins/homeserver/dendrite";
+import { Credentials } from "../../plugins/homeserver";
 
-const username = "user1234";
-// this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen.
-const password = "oETo7MPf0o";
 const email = "[email protected]";
 
+const test = base.extend<{ credentials: Pick<Credentials, "username" | "password"> }>({
+    // eslint-disable-next-line no-empty-pattern
+    credentials: async ({}, use, testInfo) => {
+        await use({
+            username: `user_${testInfo.testId}`,
+            // this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen.
+            password: "oETo7MPf0o",
+        });
+    },
+});
+
 test.use(emailHomeserver);
 test.use({
     config: {
@@ -45,31 +54,35 @@ test.describe("Forgot Password", () => {
         await expect(page.getByRole("main")).toMatchScreenshot("forgot-password.png");
     });
 
-    test("renders email verification dialog properly", { tag: "@screenshot" }, async ({ page, homeserver }) => {
-        const user = await homeserver.registerUser(username, password);
+    test(
+        "renders email verification dialog properly",
+        { tag: "@screenshot" },
+        async ({ page, homeserver, credentials }) => {
+            const user = await homeserver.registerUser(credentials.username, credentials.password);
 
-        await homeserver.setThreepid(user.userId, "email", email);
+            await homeserver.setThreepid(user.userId, "email", email);
 
-        await page.goto("/");
+            await page.goto("/");
 
-        await page.getByRole("link", { name: "Sign in" }).click();
-        await selectHomeserver(page, homeserver.baseUrl);
+            await page.getByRole("link", { name: "Sign in" }).click();
+            await selectHomeserver(page, homeserver.baseUrl);
 
-        await page.getByRole("button", { name: "Forgot password?" }).click();
+            await page.getByRole("button", { name: "Forgot password?" }).click();
 
-        await page.getByRole("textbox", { name: "Email address" }).fill(email);
+            await page.getByRole("textbox", { name: "Email address" }).fill(email);
 
-        await page.getByRole("button", { name: "Send email" }).click();
+            await page.getByRole("button", { name: "Send email" }).click();
 
-        await page.getByRole("button", { name: "Next" }).click();
+            await page.getByRole("button", { name: "Next" }).click();
 
-        await page.getByRole("textbox", { name: "New Password", exact: true }).fill(password);
-        await page.getByRole("textbox", { name: "Confirm new password", exact: true }).fill(password);
+            await page.getByRole("textbox", { name: "New Password", exact: true }).fill(credentials.password);
+            await page.getByRole("textbox", { name: "Confirm new password", exact: true }).fill(credentials.password);
 
-        await page.getByRole("button", { name: "Reset password" }).click();
+            await page.getByRole("button", { name: "Reset password" }).click();
 
-        await expect(page.getByRole("button", { name: "Resend" })).toBeInViewport();
+            await expect(page.getByRole("button", { name: "Resend" })).toBeInViewport();
 
-        await expect(page.locator(".mx_Dialog")).toMatchScreenshot("forgot-password-verify-email.png");
-    });
+            await expect(page.locator(".mx_Dialog")).toMatchScreenshot("forgot-password-verify-email.png");
+        },
+    );
 });

playwright/e2e/login/login-consent.spec.ts

@@ -77,6 +77,9 @@ async function login(page: Page, homeserver: HomeserverInstance, credentials: Cr
     await page.getByRole("button", { name: "Sign in" }).click();
 }
 
+// This test suite uses the same userId for all tests in the suite
+// due to DEVICE_SIGNING_KEYS_BODY being specific to that userId,
+// so we restart the Synapse container to make it forget everything.
 test.use(consentHomeserver);
 test.use({
     config: {
@@ -97,6 +100,9 @@ test.use({
             ...credentials,
             displayName,
         });
+
+        // Restart the homeserver to wipe its in-memory db so we can reuse the same user ID without cross-signing prompts
+        await homeserver.restart();
     },
 });
 

playwright/e2e/login/login-sso.spec.ts

@@ -17,13 +17,13 @@ test.use(legacyOAuthHomeserver);
 test.describe("SSO login", () => {
     test.skip(isDendrite, "does not yet support SSO");
 
-    test("logs in with SSO and lands on the home screen", async ({ page, homeserver }) => {
+    test("logs in with SSO and lands on the home screen", async ({ page, homeserver }, testInfo) => {
         // If this test fails with a screen showing "Timeout connecting to remote server", it is most likely due to
         // your firewall settings: Synapse is unable to reach the OIDC server.
         //
         // If you are using ufw, try something like:
         //    sudo ufw allow in on docker0
         //
-        await doTokenRegistration(page, homeserver);
+        await doTokenRegistration(page, homeserver, testInfo);
     });
 });

playwright/e2e/login/soft_logout_oauth.spec.ts

@@ -26,8 +26,8 @@ test.use({
 test.use(legacyOAuthHomeserver);
 test.describe("Soft logout with SSO user", () => {
     test.use({
-        user: async ({ page, homeserver }, use) => {
-            const user = await doTokenRegistration(page, homeserver);
+        user: async ({ page, homeserver }, use, testInfo) => {
+            const user = await doTokenRegistration(page, homeserver, testInfo);
 
             // Eventually, we should end up at the home screen.
             await expect(page).toHaveURL(/\/#\/home$/);

playwright/e2e/login/utils.ts

@@ -6,7 +6,7 @@
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Page, expect } from "@playwright/test";
+import { Page, expect, TestInfo } from "@playwright/test";
 
 import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
 
@@ -15,6 +15,7 @@
 export async function doTokenRegistration(
     page: Page,
     homeserver: HomeserverInstance,
+    testInfo: TestInfo,
 ): Promise<Credentials & { displayName: string }> {
     await page.goto("/#/login");
 
@@ -34,8 +35,8 @@
     await page.getByRole("button", { name: "Submit" }).click();
 
     // Synapse prompts us to pick a user ID
     await expect(page.getByRole("heading", { name: "Create your account" })).toBeVisible();
-    await page.getByRole("textbox", { name: "Username (required)" }).fill("alice");
+    await page.getByRole("textbox", { name: "Username (required)" }).fill(`alice_${testInfo.testId}`);
 
     // wait for username validation to start, and complete
     await expect(page.locator("#field-username-output")).toHaveText("");

playwright/e2e/oidc/index.ts

@@ -31,9 +31,9 @@
     await expect(async () => {
         messages = await mailhog.messages();
         expect(messages.items).toHaveLength(1);
     }).toPass();
     expect(messages.items[0].to).toEqual(`${username} <${email}>`);
-    const [code] = messages.items[0].text.match(/(\d{6})/);
+    const [, code] = messages.items[0].text.match(/Your verification code to confirm this email address is: (\d{6})/);
 
     await page.getByRole("textbox", { name: "6-digit code" }).fill(code);
     await page.getByRole("button", { name: "Continue" }).click();

playwright/e2e/oidc/oidc-native.spec.ts

@@ -17,19 +17,30 @@
     test.skip(isDendrite, "does not yet support MAS");
     test.slow(); // trace recording takes a while here
 
-    test("can register the oauth2 client and an account", async ({ context, page, homeserver, mailhogClient, mas }) => {
+    test("can register the oauth2 client and an account", async ({
+        context,
+        page,
+        homeserver,
+        mailhogClient,
+        mas,
+    }, testInfo) => {
+        await page.clock.install();
+
         const tokenUri = `${mas.baseUrl}/oauth2/token`;
         const tokenApiPromise = page.waitForRequest(
             (request) => request.url() === tokenUri && request.postDataJSON()["grant_type"] === "authorization_code",
         );
 
         await page.goto("/#/login");
         await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "[email protected]", "Pa$sW0rD!");
+
+        const userId = `alice_${testInfo.testId}`;
+        await registerAccountMas(page, mailhogClient, userId, "[email protected]", "Pa$sW0rD!");
 
         // Eventually, we should end up at the home screen.
         await expect(page).toHaveURL(/\/#\/home$/, { timeout: 10000 });
-        await expect(page.getByRole("heading", { name: "Welcome alice", exact: true })).toBeVisible();
+        await expect(page.getByRole("heading", { name: `Welcome ${userId}`, exact: true })).toBeVisible();
+        await page.clock.runFor(20000); // run the timer so we see the token request
 
         const tokenApiRequest = await tokenApiPromise;
         expect(tokenApiRequest.postDataJSON()["grant_type"]).toBe("authorization_code");

playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts

@@ -17,8 +17,8 @@ const test = base.extend<{
 test.describe("1:1 chat room", () => {
     test.use({
         displayName: "Jeff",
-        user2: async ({ homeserver }, use) => {
-            const credentials = await homeserver.registerUser("user1234", "p4s5W0rD", "Timmy");
+        user2: async ({ homeserver }, use, testInfo) => {
+            const credentials = await homeserver.registerUser(`user2_${testInfo.testId}`, "p4s5W0rD", "Timmy");
             await use(credentials);
         },
     });

playwright/e2e/share-dialog/share-dialog.spec.ts

@@ -23,7 +23,7 @@
 
         const dialog = page.getByRole("dialog", { name: "Share room" });
         await expect(dialog.getByText(`https://matrix.to/#/${room.roomId}`)).toBeVisible();
-        expect(dialog).toMatchScreenshot("share-dialog-room.png", {
+        await expect(dialog).toMatchScreenshot("share-dialog-room.png", {
             // QRCode and url changes at every run
             mask: [page.locator(".mx_QRCode"), page.locator(".mx_ShareDialog_top > span")],
         });
@@ -40,7 +40,7 @@
 
         const dialog = page.getByRole("dialog", { name: "Share User" });
         await expect(dialog.getByText(`https://matrix.to/#/${user.userId}`)).toBeVisible();
-        expect(dialog).toMatchScreenshot("share-dialog-user.png", {
+        await expect(dialog).toMatchScreenshot("share-dialog-user.png", {
             // QRCode changes at every run
             mask: [page.locator(".mx_QRCode")],
         });
@@ -57,7 +57,7 @@
 
         const dialog = page.getByRole("dialog", { name: "Share Room Message" });
         await expect(dialog.getByRole("checkbox", { name: "Link to selected message" })).toBeChecked();
-        expect(dialog).toMatchScreenshot("share-dialog-event.png", {
+        await expect(dialog).toMatchScreenshot("share-dialog-event.png", {
             // QRCode and url changes at every run
             mask: [page.locator(".mx_QRCode"), page.locator(".mx_ShareDialog_top > span")],
         });

playwright/e2e/sliding-sync/sliding-sync.spec.ts

@@ -83,7 +83,7 @@
     };
 
     // Load the user fixture for all tests
     test.beforeEach(({ user }) => {});
 
     test("should render the Rooms list in reverse chronological order by default and allowing sorting A-Z", async ({
         page,
@@ -108,7 +108,6 @@
         await page.getByRole("menuitemradio", { name: "A-Z" }).dispatchEvent("click");
         await expect(page.locator(".mx_StyledRadioButton_checked").getByText("A-Z")).toBeVisible();
 
-        await page.pause();
         await checkOrder(["Apple", "Orange", "Pineapple", "Test Room"], page);
     });