feat: add support for flaky tests #4

playwright.config.ts

@@ -26,7 +26,7 @@ const config: PlaywrightTestConfig<{}, {}> = {
         linkUrlOnFailure:
           "https://github.com/playwright-community/playwright-msteams-reporter/issues",
         linkTextOnFailure: "Report an issue",
-        mentionOnFailure: "Elio <[email protected]>, [email protected]",
+        mentionOnFailure: "[email protected]",
         mentionOnFailureText: "",
         debug: true,
       },

src/constants/Images.ts

@@ -1,4 +1,5 @@
 export const Images = {
   success: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAFCAIAAAAL5hHIAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAEklEQVQImWOweTCViYGBARkDAB59Abohe/o4AAAAAElFTkSuQmCC`,
+  flaky: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAFCAIAAAAL5hHIAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAEUlEQVQImWP4nszDxMDAgIwBGnsBb7ZK8egAAAAASUVORK5CYII=`,
   failed: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAFCAIAAAAL5hHIAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAEUlEQVQImWP45+XJxMDAgIwBHUsBmph35dMAAAAASUVORK5CYII=`,
 };

src/models/Table.ts

@@ -19,7 +19,7 @@ export interface TableCell {
   style?: TableCellStyle;
 }
 
-export type TableCellStyle = "attention" | "good" | "warning";
+export type TableCellStyle = "attention" | "good" | "warning" | "accent";
 
 export interface TextBlock {
   type: "TextBlock";

src/models/TestStatuses.ts

@@ -0,0 +1,6 @@
+export interface TestStatuses {
+  passed: number;
+  flaky: number;
+  failed: number;
+  skipped: number;
+}

src/models/index.ts

@@ -1,3 +1,4 @@
 export * from "./AdaptiveCard";
 export * from "./Table";
+export * from "./TestStatuses";
 export * from "./WebhookType";

src/processResults.test.ts

@@ -20,27 +20,45 @@ const DEFAULT_OPTIONS: MsTeamsReporterOptions = {
 const SUITE_MOCK_PASSED = {
   suites: [
     {
-      allTests: () => [{ results: [{ status: "passed" }] }],
+      allTests: () => [{ outcome: () => "expected" }],
     },
     {
       allTests: () => [
-        { results: [{ status: "passed" }] },
-        { results: [{ status: "passed" }] },
+        { outcome: () => "expected" },
+        { outcome: () => "expected" },
       ],
     },
   ],
   allTests: () => [{}, {}, {}],
 };
 
+const SUITE_MOCK_FLAKY = {
+  suites: [
+    {
+      allTests: () => [{ outcome: () => "expected" }],
+    },
+    {
+      allTests: () => [
+        { outcome: () => "expected" },
+        { outcome: () => "expected" },
+      ],
+    },
+    {
+      allTests: () => [{ outcome: () => "flaky" }],
+    },
+  ],
+  allTests: () => [{}, {}, {}],
+};
+
 const SUITE_MOCK_FAILED = {
   suites: [
     {
-      allTests: () => [{ results: [{ status: "failed" }] }],
+      allTests: () => [{ outcome: () => "unexpected" }],
     },
     {
       allTests: () => [
-        { results: [{ status: "passed" }] },
-        { results: [{ status: "passed" }] },
+        { outcome: () => "expected" },
+        { outcome: () => "expected" },
       ],
     },
   ],
@@ -165,6 +183,30 @@ describe("processResults", () => {
     consoleErrorSpy.mockReset();
   });
 
+  it("should include a flaky row", async () => {
+    const consoleLogSpy = jest
+      .spyOn(console, "log")
+      .mockImplementation((message) => {
+        if (message.includes("message") && message.includes("Flaky")) {
+          console.log(`Flaky`);
+        }
+      });
+    const fetchMock = jest
+      .fn()
+      .mockResolvedValue({ ok: true, text: () => "1" });
+    global.fetch = fetchMock;
+    const options: MsTeamsReporterOptions = {
+      ...DEFAULT_OPTIONS,
+      webhookUrl: FLOW_WEBHOOK_URL,
+      webhookType: "powerautomate",
+      debug: true,
+    };
+    await processResults(SUITE_MOCK_FLAKY as any, options);
+    expect(consoleLogSpy).toHaveBeenCalledWith("Flaky");
+
+    consoleLogSpy.mockReset();
+  });
+
   it("should use version 1.4 for adaptive card", async () => {
     const consoleLogSpy = jest
       .spyOn(console, "log")
@@ -277,6 +319,39 @@ describe("processResults", () => {
     consoleLogSpy.mockReset();
   });
 
+  it("should include the failure link", async () => {
+    const fakeFailureLink =
+      "https://github.com/estruyf/playwright-msteams-reporter";
+    const fakeFailureText = "View the failed tests";
+    const consoleLogSpy = jest
+      .spyOn(console, "log")
+      .mockImplementation((message) => {
+        if (
+          message.includes("message") &&
+          message.includes(fakeFailureLink) &&
+          message.includes(fakeFailureText)
+        ) {
+          console.log(fakeFailureText);
+        }
+      });
+    const fetchMock = jest
+      .fn()
+      .mockResolvedValue({ ok: true, text: () => "1" });
+    global.fetch = fetchMock;
+    const options: MsTeamsReporterOptions = {
+      ...DEFAULT_OPTIONS,
+      webhookUrl: FLOW_WEBHOOK_URL,
+      webhookType: "powerautomate",
+      linkUrlOnFailure: fakeFailureLink,
+      linkTextOnFailure: "View the failed tests",
+      debug: true,
+    };
+    await processResults(SUITE_MOCK_FAILED as any, options);
+    expect(consoleLogSpy).toHaveBeenCalledWith(fakeFailureText);
+
+    consoleLogSpy.mockReset();
+  });
+
   it("should show debug message", async () => {
     const consoleLogSpy = jest.spyOn(console, "log").mockImplementation();
     const fetchMock = jest

src/processResults.ts

@@ -3,10 +3,13 @@ import { MsTeamsReporterOptions } from ".";
 import {
   createTableRow,
   getMentions,
+  getNotificationBackground,
+  getNotificationColor,
+  getNotificationTitle,
   getTotalStatus,
   validateWebhookUrl,
 } from "./utils";
-import { BaseAdaptiveCard, BaseTable, Images } from "./constants";
+import { BaseAdaptiveCard, BaseTable } from "./constants";
 
 export const processResults = async (
   suite: Suite | undefined,
@@ -33,8 +36,7 @@ export const processResults = async (
 
   const totalStatus = getTotalStatus(suite.suites);
   const totalTests = suite.allTests().length;
-  const failedTests = totalStatus.failed + totalStatus.timedOut;
-  const isSuccess = failedTests === 0;
+  const isSuccess = totalStatus.failed === 0;
 
   if (isSuccess && !options.notifyOnSuccess) {
     if (!options.quiet) {
@@ -48,11 +50,16 @@ export const processResults = async (
   table.rows.push(
     createTableRow("Passed", totalStatus.passed, { style: "good" })
   );
+  if (totalStatus.flaky) {
+    table.rows.push(
+      createTableRow("Flaky", totalStatus.flaky, { style: "warning" })
+    );
+  }
   table.rows.push(
-    createTableRow("Failed", failedTests, { style: "attention" })
+    createTableRow("Failed", totalStatus.failed, { style: "attention" })
   );
   table.rows.push(
-    createTableRow("Skipped", totalStatus.skipped, { style: "warning" })
+    createTableRow("Skipped", totalStatus.skipped, { style: "accent" })
   );
   table.rows.push(
     createTableRow("Total tests", totalTests, {
@@ -74,14 +81,14 @@ export const processResults = async (
         type: "TextBlock",
         size: "Large",
         weight: "Bolder",
-        text: isSuccess ? "Tests passed" : "Tests failed",
-        color: isSuccess ? "Good" : "Attention",
+        text: getNotificationTitle(totalStatus),
+        color: getNotificationColor(totalStatus),
       },
       table,
     ] as any[],
     bleed: true,
     backgroundImage: {
-      url: isSuccess ? Images.success : Images.failed,
+      url: getNotificationBackground(totalStatus),
       fillMode: "RepeatHorizontally",
     },
   };

src/utils/getNotificationBackground.test.ts

@@ -0,0 +1,44 @@
+import { TestStatuses } from "../models";
+import { getNotificationBackground } from ".";
+import { Images } from "../constants";
+
+describe("getNotificationBackground", () => {
+  it("Should return 'success' background", () => {
+    const statuses: TestStatuses = {
+      passed: 1,
+      failed: 0,
+      flaky: 0,
+      skipped: 0,
+    };
+
+    const title = getNotificationBackground(statuses);
+
+    expect(title).toBe(Images.success);
+  });
+
+  it("Should return 'flaky' background", () => {
+    const statuses: TestStatuses = {
+      passed: 1,
+      failed: 0,
+      flaky: 1,
+      skipped: 0,
+    };
+
+    const title = getNotificationBackground(statuses);
+
+    expect(title).toBe(Images.flaky);
+  });
+
+  it("Should return 'failed' background", () => {
+    const statuses: TestStatuses = {
+      passed: 1,
+      failed: 1,
+      flaky: 1,
+      skipped: 0,
+    };
+
+    const title = getNotificationBackground(statuses);
+
+    expect(title).toBe(Images.failed);
+  });
+});

src/utils/getNotificationBackground.ts

@@ -0,0 +1,17 @@
+import { TestStatuses } from "../models";
+import { getNotificationOutcome } from ".";
+import { Images } from "../constants";
+
+export const getNotificationBackground = (statuses: TestStatuses) => {
+  const outcome = getNotificationOutcome(statuses);
+
+  if (outcome === "passed") {
+    return Images.success;
+  }
+
+  if (outcome === "flaky") {
+    return Images.flaky;
+  }
+
+  return Images.failed;
+};

src/utils/getNotificationColor.test.ts

@@ -0,0 +1,43 @@
+import { TestStatuses } from "../models";
+import { getNotificationColor } from ".";
+
+describe("getNotificationColor", () => {
+  it("Should return 'good' background", () => {
+    const statuses: TestStatuses = {
+      passed: 1,
+      failed: 0,
+      flaky: 0,
+      skipped: 0,
+    };
+
+    const title = getNotificationColor(statuses);
+
+    expect(title).toBe("Good");
+  });
+
+  it("Should return 'warning' background", () => {
+    const statuses: TestStatuses = {
+      passed: 1,
+      failed: 0,
+      flaky: 1,
+      skipped: 0,
+    };
+
+    const title = getNotificationColor(statuses);
+
+    expect(title).toBe("Warning");
+  });
+
+  it("Should return 'attention' background", () => {
+    const statuses: TestStatuses = {
+      passed: 1,
+      failed: 1,
+      flaky: 1,
+      skipped: 0,
+    };
+
+    const title = getNotificationColor(statuses);
+
+    expect(title).toBe("Attention");
+  });
+});

src/utils/getNotificationColor.ts

@@ -0,0 +1,18 @@
+import { TestStatuses } from "../models";
+import { getNotificationOutcome } from ".";
+
+export const getNotificationColor = (
+  statuses: TestStatuses
+): "Good" | "Warning" | "Attention" => {
+  const outcome = getNotificationOutcome(statuses);
+
+  if (outcome === "passed") {
+    return "Good";
+  }
+
+  if (outcome === "flaky") {
+    return "Warning";
+  }
+
+  return "Attention";
+};

src/utils/getNotificationOutcome.ts

@@ -0,0 +1,18 @@
+import { TestStatuses } from "../models";
+
+export const getNotificationOutcome = (
+  statuses: TestStatuses
+): "passed" | "flaky" | "failed" => {
+  const isSuccess = statuses.failed === 0;
+  const hasFlakyTests = statuses.flaky > 0;
+
+  if (isSuccess && !hasFlakyTests) {
+    return "passed";
+  }
+
+  if (isSuccess && hasFlakyTests) {
+    return "flaky";
+  }
+
+  return "failed";
+};