diff --git a/playwright.config.ts b/playwright.config.ts index c6f0740..2890fc6 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -26,7 +26,7 @@ const config: PlaywrightTestConfig<{}, {}> = { linkUrlOnFailure: "https://github.com/playwright-community/playwright-msteams-reporter/issues", linkTextOnFailure: "Report an issue", - mentionOnFailure: "Elio , mail@elio.dev", + mentionOnFailure: "elio@struyfconsulting.be", mentionOnFailureText: "", debug: true, }, diff --git a/src/constants/Images.ts b/src/constants/Images.ts index 29510bf..4d638ee 100644 --- a/src/constants/Images.ts +++ b/src/constants/Images.ts @@ -1,4 +1,5 @@ export const Images = { success: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAFCAIAAAAL5hHIAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAEklEQVQImWOweTCViYGBARkDAB59Abohe/o4AAAAAElFTkSuQmCC`, + flaky: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAFCAIAAAAL5hHIAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAEUlEQVQImWP4nszDxMDAgIwBGnsBb7ZK8egAAAAASUVORK5CYII=`, failed: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAFCAIAAAAL5hHIAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAEUlEQVQImWP45+XJxMDAgIwBHUsBmph35dMAAAAASUVORK5CYII=`, }; diff --git a/src/models/Table.ts b/src/models/Table.ts index bfe7dff..2afb4f8 100644 --- a/src/models/Table.ts +++ b/src/models/Table.ts @@ -19,7 +19,7 @@ export interface TableCell { style?: TableCellStyle; } -export type TableCellStyle = "attention" | "good" | "warning"; +export type TableCellStyle = "attention" | "good" | "warning" | "accent"; export interface TextBlock { type: "TextBlock"; diff --git a/src/models/TestStatuses.ts b/src/models/TestStatuses.ts new file mode 100644 index 0000000..8b73071 --- /dev/null +++ b/src/models/TestStatuses.ts @@ -0,0 +1,6 @@ +export interface TestStatuses { + passed: number; + flaky: number; + failed: number; + skipped: number; +} diff --git a/src/models/index.ts b/src/models/index.ts index f399799..067e7a8 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1,3 +1,4 @@ export * from "./AdaptiveCard"; export * from "./Table"; +export * from "./TestStatuses"; export * from "./WebhookType"; diff --git a/src/processResults.test.ts b/src/processResults.test.ts index 0064b92..535f402 100644 --- a/src/processResults.test.ts +++ b/src/processResults.test.ts @@ -20,27 +20,45 @@ const DEFAULT_OPTIONS: MsTeamsReporterOptions = { const SUITE_MOCK_PASSED = { suites: [ { - allTests: () => [{ results: [{ status: "passed" }] }], + allTests: () => [{ outcome: () => "expected" }], }, { allTests: () => [ - { results: [{ status: "passed" }] }, - { results: [{ status: "passed" }] }, + { outcome: () => "expected" }, + { outcome: () => "expected" }, ], }, ], allTests: () => [{}, {}, {}], }; +const SUITE_MOCK_FLAKY = { + suites: [ + { + allTests: () => [{ outcome: () => "expected" }], + }, + { + allTests: () => [ + { outcome: () => "expected" }, + { outcome: () => "expected" }, + ], + }, + { + allTests: () => [{ outcome: () => "flaky" }], + }, + ], + allTests: () => [{}, {}, {}], +}; + const SUITE_MOCK_FAILED = { suites: [ { - allTests: () => [{ results: [{ status: "failed" }] }], + allTests: () => [{ outcome: () => "unexpected" }], }, { allTests: () => [ - { results: [{ status: "passed" }] }, - { results: [{ status: "passed" }] }, + { outcome: () => "expected" }, + { outcome: () => "expected" }, ], }, ], @@ -165,6 +183,30 @@ describe("processResults", () => { consoleErrorSpy.mockReset(); }); + it("should include a flaky row", async () => { + const consoleLogSpy = jest + .spyOn(console, "log") + .mockImplementation((message) => { + if (message.includes("message") && message.includes("Flaky")) { + console.log(`Flaky`); + } + }); + const fetchMock = jest + .fn() + .mockResolvedValue({ ok: true, text: () => "1" }); + global.fetch = fetchMock; + const options: MsTeamsReporterOptions = { + ...DEFAULT_OPTIONS, + webhookUrl: FLOW_WEBHOOK_URL, + webhookType: "powerautomate", + debug: true, + }; + await processResults(SUITE_MOCK_FLAKY as any, options); + expect(consoleLogSpy).toHaveBeenCalledWith("Flaky"); + + consoleLogSpy.mockReset(); + }); + it("should use version 1.4 for adaptive card", async () => { const consoleLogSpy = jest .spyOn(console, "log") @@ -277,6 +319,39 @@ describe("processResults", () => { consoleLogSpy.mockReset(); }); + it("should include the failure link", async () => { + const fakeFailureLink = + "https://github.com/estruyf/playwright-msteams-reporter"; + const fakeFailureText = "View the failed tests"; + const consoleLogSpy = jest + .spyOn(console, "log") + .mockImplementation((message) => { + if ( + message.includes("message") && + message.includes(fakeFailureLink) && + message.includes(fakeFailureText) + ) { + console.log(fakeFailureText); + } + }); + const fetchMock = jest + .fn() + .mockResolvedValue({ ok: true, text: () => "1" }); + global.fetch = fetchMock; + const options: MsTeamsReporterOptions = { + ...DEFAULT_OPTIONS, + webhookUrl: FLOW_WEBHOOK_URL, + webhookType: "powerautomate", + linkUrlOnFailure: fakeFailureLink, + linkTextOnFailure: "View the failed tests", + debug: true, + }; + await processResults(SUITE_MOCK_FAILED as any, options); + expect(consoleLogSpy).toHaveBeenCalledWith(fakeFailureText); + + consoleLogSpy.mockReset(); + }); + it("should show debug message", async () => { const consoleLogSpy = jest.spyOn(console, "log").mockImplementation(); const fetchMock = jest diff --git a/src/processResults.ts b/src/processResults.ts index 01b6b55..53938b6 100644 --- a/src/processResults.ts +++ b/src/processResults.ts @@ -3,10 +3,13 @@ import { MsTeamsReporterOptions } from "."; import { createTableRow, getMentions, + getNotificationBackground, + getNotificationColor, + getNotificationTitle, getTotalStatus, validateWebhookUrl, } from "./utils"; -import { BaseAdaptiveCard, BaseTable, Images } from "./constants"; +import { BaseAdaptiveCard, BaseTable } from "./constants"; export const processResults = async ( suite: Suite | undefined, @@ -33,8 +36,7 @@ export const processResults = async ( const totalStatus = getTotalStatus(suite.suites); const totalTests = suite.allTests().length; - const failedTests = totalStatus.failed + totalStatus.timedOut; - const isSuccess = failedTests === 0; + const isSuccess = totalStatus.failed === 0; if (isSuccess && !options.notifyOnSuccess) { if (!options.quiet) { @@ -48,11 +50,16 @@ export const processResults = async ( table.rows.push( createTableRow("Passed", totalStatus.passed, { style: "good" }) ); + if (totalStatus.flaky) { + table.rows.push( + createTableRow("Flaky", totalStatus.flaky, { style: "warning" }) + ); + } table.rows.push( - createTableRow("Failed", failedTests, { style: "attention" }) + createTableRow("Failed", totalStatus.failed, { style: "attention" }) ); table.rows.push( - createTableRow("Skipped", totalStatus.skipped, { style: "warning" }) + createTableRow("Skipped", totalStatus.skipped, { style: "accent" }) ); table.rows.push( createTableRow("Total tests", totalTests, { @@ -74,14 +81,14 @@ export const processResults = async ( type: "TextBlock", size: "Large", weight: "Bolder", - text: isSuccess ? "Tests passed" : "Tests failed", - color: isSuccess ? "Good" : "Attention", + text: getNotificationTitle(totalStatus), + color: getNotificationColor(totalStatus), }, table, ] as any[], bleed: true, backgroundImage: { - url: isSuccess ? Images.success : Images.failed, + url: getNotificationBackground(totalStatus), fillMode: "RepeatHorizontally", }, }; diff --git a/src/utils/getNotificationBackground.test.ts b/src/utils/getNotificationBackground.test.ts new file mode 100644 index 0000000..07564c6 --- /dev/null +++ b/src/utils/getNotificationBackground.test.ts @@ -0,0 +1,44 @@ +import { TestStatuses } from "../models"; +import { getNotificationBackground } from "."; +import { Images } from "../constants"; + +describe("getNotificationBackground", () => { + it("Should return 'success' background", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 0, + flaky: 0, + skipped: 0, + }; + + const title = getNotificationBackground(statuses); + + expect(title).toBe(Images.success); + }); + + it("Should return 'flaky' background", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 0, + flaky: 1, + skipped: 0, + }; + + const title = getNotificationBackground(statuses); + + expect(title).toBe(Images.flaky); + }); + + it("Should return 'failed' background", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 1, + flaky: 1, + skipped: 0, + }; + + const title = getNotificationBackground(statuses); + + expect(title).toBe(Images.failed); + }); +}); diff --git a/src/utils/getNotificationBackground.ts b/src/utils/getNotificationBackground.ts new file mode 100644 index 0000000..f8a4ba3 --- /dev/null +++ b/src/utils/getNotificationBackground.ts @@ -0,0 +1,17 @@ +import { TestStatuses } from "../models"; +import { getNotificationOutcome } from "."; +import { Images } from "../constants"; + +export const getNotificationBackground = (statuses: TestStatuses) => { + const outcome = getNotificationOutcome(statuses); + + if (outcome === "passed") { + return Images.success; + } + + if (outcome === "flaky") { + return Images.flaky; + } + + return Images.failed; +}; diff --git a/src/utils/getNotificationColor.test.ts b/src/utils/getNotificationColor.test.ts new file mode 100644 index 0000000..5996889 --- /dev/null +++ b/src/utils/getNotificationColor.test.ts @@ -0,0 +1,43 @@ +import { TestStatuses } from "../models"; +import { getNotificationColor } from "."; + +describe("getNotificationColor", () => { + it("Should return 'good' background", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 0, + flaky: 0, + skipped: 0, + }; + + const title = getNotificationColor(statuses); + + expect(title).toBe("Good"); + }); + + it("Should return 'warning' background", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 0, + flaky: 1, + skipped: 0, + }; + + const title = getNotificationColor(statuses); + + expect(title).toBe("Warning"); + }); + + it("Should return 'attention' background", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 1, + flaky: 1, + skipped: 0, + }; + + const title = getNotificationColor(statuses); + + expect(title).toBe("Attention"); + }); +}); diff --git a/src/utils/getNotificationColor.ts b/src/utils/getNotificationColor.ts new file mode 100644 index 0000000..bc10dc9 --- /dev/null +++ b/src/utils/getNotificationColor.ts @@ -0,0 +1,18 @@ +import { TestStatuses } from "../models"; +import { getNotificationOutcome } from "."; + +export const getNotificationColor = ( + statuses: TestStatuses +): "Good" | "Warning" | "Attention" => { + const outcome = getNotificationOutcome(statuses); + + if (outcome === "passed") { + return "Good"; + } + + if (outcome === "flaky") { + return "Warning"; + } + + return "Attention"; +}; diff --git a/src/utils/getNotificationOutcome.ts b/src/utils/getNotificationOutcome.ts new file mode 100644 index 0000000..38022c1 --- /dev/null +++ b/src/utils/getNotificationOutcome.ts @@ -0,0 +1,18 @@ +import { TestStatuses } from "../models"; + +export const getNotificationOutcome = ( + statuses: TestStatuses +): "passed" | "flaky" | "failed" => { + const isSuccess = statuses.failed === 0; + const hasFlakyTests = statuses.flaky > 0; + + if (isSuccess && !hasFlakyTests) { + return "passed"; + } + + if (isSuccess && hasFlakyTests) { + return "flaky"; + } + + return "failed"; +}; diff --git a/src/utils/getNotificationTitle.test.ts b/src/utils/getNotificationTitle.test.ts new file mode 100644 index 0000000..eeca353 --- /dev/null +++ b/src/utils/getNotificationTitle.test.ts @@ -0,0 +1,56 @@ +import { TestStatuses } from "../models"; +import { getNotificationTitle } from "."; + +describe("getNotificationTitle", () => { + it("should return 'Tests passed' when the outcome is 'passed'", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 0, + flaky: 0, + skipped: 0, + }; + + const title = getNotificationTitle(statuses); + + expect(title).toBe("Tests passed"); + }); + + it("should return 'Tests passed with flaky tests' when the outcome is 'flaky'", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 0, + flaky: 1, + skipped: 0, + }; + + const title = getNotificationTitle(statuses); + + expect(title).toBe("Tests passed with flaky tests"); + }); + + it("should return 'Tests failed' when the outcome is neither 'passed' nor 'flaky'", () => { + const statuses: TestStatuses = { + passed: 1, + failed: 1, + flaky: 1, + skipped: 0, + }; + + const title = getNotificationTitle(statuses); + + expect(title).toBe("Tests failed"); + }); + + it("should return 'Tests passed' when only skipped", () => { + const statuses: TestStatuses = { + passed: 0, + failed: 0, + flaky: 0, + skipped: 1, + }; + + const title = getNotificationTitle(statuses); + + expect(title).toBe("Tests passed"); + }); +}); diff --git a/src/utils/getNotificationTitle.ts b/src/utils/getNotificationTitle.ts new file mode 100644 index 0000000..322d3c4 --- /dev/null +++ b/src/utils/getNotificationTitle.ts @@ -0,0 +1,16 @@ +import { TestStatuses } from "../models"; +import { getNotificationOutcome } from "."; + +export const getNotificationTitle = (statuses: TestStatuses): string => { + const outcome = getNotificationOutcome(statuses); + + if (outcome === "passed") { + return "Tests passed"; + } + + if (outcome === "flaky") { + return "Tests passed with flaky tests"; + } + + return "Tests failed"; +}; diff --git a/src/utils/getTestOutcome.test.ts b/src/utils/getTestOutcome.test.ts deleted file mode 100644 index 2105f33..0000000 --- a/src/utils/getTestOutcome.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { getTestOutcome } from "./getTestOutcome"; - -describe("getTestOutcome", () => { - it("should return the status if it exists", () => { - const test: any = { - outcome: () => "expected", - }; - const result: any = { - status: "passed", - }; - - const outcome = getTestOutcome(test, result); - - expect(outcome).toBe("passed"); - }); - - it("should return the result status if it exists", () => { - const test: any = { - outcome: () => "expected", - }; - const result: any = { - status: "passed", - }; - - const outcome = getTestOutcome(test, result); - - expect(outcome).toBe("passed"); - }); - - it("should return 'passed' when the test outcome is 'expected'", () => { - const test: any = { - outcome: () => "expected", - }; - const result: any = {}; - - const outcome = getTestOutcome(test, result); - - expect(outcome).toBe("passed"); - }); - - it("should return 'failed' when the test outcome is 'flaky'", () => { - const test: any = { - outcome: () => "flaky", - }; - const result: any = {}; - - const outcome = getTestOutcome(test, result); - - expect(outcome).toBe("failed"); - }); - - it("should return 'failed' when the test outcome is 'unexpected'", () => { - const test: any = { - outcome: () => "unexpected", - }; - const result: any = {}; - - const outcome = getTestOutcome(test, result); - - expect(outcome).toBe("failed"); - }); - - it("should return the test outcome when it is neither 'expected', 'flaky', nor 'unexpected'", () => { - const test: any = { - outcome: () => "other", - }; - const result: any = {}; - - const outcome = getTestOutcome(test, result); - - expect(outcome).toBe("other"); - }); -}); diff --git a/src/utils/getTestOutcome.ts b/src/utils/getTestOutcome.ts deleted file mode 100644 index 5c8e7da..0000000 --- a/src/utils/getTestOutcome.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { TestCase, TestResult } from "@playwright/test/reporter"; - -export const getTestOutcome = (test: TestCase, result: TestResult) => { - if (result?.status) { - return result.status; - } - const testOutcome = test.outcome(); - switch (testOutcome) { - case "expected": - return "passed"; - case "flaky": - case "unexpected": - return "failed"; - default: - return testOutcome; - } -}; diff --git a/src/utils/getTotalStatus.test.ts b/src/utils/getTotalStatus.test.ts index 487553a..5506acc 100644 --- a/src/utils/getTotalStatus.test.ts +++ b/src/utils/getTotalStatus.test.ts @@ -19,12 +19,8 @@ describe("getTotalStatus", () => { ...baseSuite, allTests: () => [ - { - results: [{ status: "passed" }], - }, - { - results: [{ status: "passed" }], - }, + { outcome: () => "expected" }, + { outcome: () => "expected" }, ] as any[], }, ]; @@ -34,33 +30,46 @@ describe("getTotalStatus", () => { expect(result).toEqual({ passed: 2, failed: 0, + flaky: 0, skipped: 0, - timedOut: 0, }); }); - it("should return the correct total status when there are failed, skipped, and timed out tests", () => { + it("should return the correct flaky total when there are flaky tests", () => { const suites: Suite[] = [ { ...baseSuite, allTests: () => [ - { - results: [{ status: "passed" }], - }, - { - results: [{ status: "failed" }], - }, - { - results: [{ status: "skipped" }], - }, - { - results: [{ status: "timedOut" }], - }, - { - results: [{}], - outcome: () => "unexpected", - }, + { outcome: () => "expected" }, + { outcome: () => "expected" }, + { outcome: () => "unexpected" }, + { outcome: () => "flaky" }, + ] as any[], + }, + ]; + + const result = getTotalStatus(suites); + + expect(result).toEqual({ + passed: 2, + failed: 1, + flaky: 1, + skipped: 0, + }); + }); + + it("should return the correct total status when there are failed, skipped tests", () => { + const suites: Suite[] = [ + { + ...baseSuite, + allTests: () => + [ + { outcome: () => "expected" }, + { outcome: () => "unexpected" }, + { outcome: () => "flaky" }, + { outcome: () => "unexpected" }, + { outcome: () => "skipped" }, ] as any[], }, ]; @@ -70,8 +79,8 @@ describe("getTotalStatus", () => { expect(result).toEqual({ passed: 1, failed: 2, + flaky: 1, skipped: 1, - timedOut: 1, }); }); @@ -83,8 +92,8 @@ describe("getTotalStatus", () => { expect(result).toEqual({ passed: 0, failed: 0, + flaky: 0, skipped: 0, - timedOut: 0, }); }); }); diff --git a/src/utils/getTotalStatus.ts b/src/utils/getTotalStatus.ts index 8672028..8a6fc1c 100644 --- a/src/utils/getTotalStatus.ts +++ b/src/utils/getTotalStatus.ts @@ -1,36 +1,28 @@ import { Suite } from "@playwright/test/reporter"; -import { getTestOutcome } from "."; +import { TestStatuses } from "../models"; -export const getTotalStatus = ( - suites: Suite[] -): { - passed: number; - failed: number; - skipped: number; - timedOut: number; -} => { +export const getTotalStatus = (suites: Suite[]): TestStatuses => { let total = { passed: 0, + flaky: 0, failed: 0, skipped: 0, - timedOut: 0, }; for (const suite of suites) { const testOutcome = suite.allTests().map((test) => { - const lastResult = test.results[test.results.length - 1]; - return getTestOutcome(test, lastResult); + return test.outcome(); }); for (const outcome of testOutcome) { - if (outcome === "passed") { + if (outcome === "expected") { total.passed++; - } else if (outcome === "failed") { + } else if (outcome === "flaky") { + total.flaky++; + } else if (outcome === "unexpected") { total.failed++; } else if (outcome === "skipped") { total.skipped++; - } else if (outcome === "timedOut") { - total.timedOut++; } } } diff --git a/src/utils/index.ts b/src/utils/index.ts index 46f433f..407f9a4 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,5 +1,8 @@ export * from "./createTableRow"; export * from "./getMentions"; -export * from "./getTestOutcome"; +export * from "./getNotificationBackground"; +export * from "./getNotificationColor"; +export * from "./getNotificationOutcome"; +export * from "./getNotificationTitle"; export * from "./getTotalStatus"; export * from "./validateWebhookUrl"; diff --git a/tests/retry.spec.ts b/tests/retry.spec.ts index ca1bb2c..d21c276 100644 --- a/tests/retry.spec.ts +++ b/tests/retry.spec.ts @@ -1,26 +1,32 @@ import { test, expect, Page } from "@playwright/test"; test.describe("Test retry", () => { - let page: Page; + test.setTimeout(10000); - test.beforeAll(async ({ browser }) => { - page = await browser.newPage(); + test("First test should fail, next should work", async ({}, testInfo) => { + if (testInfo.retry === 0) { + expect(true).toBeFalsy(); + } + expect(true).toBeTruthy(); + }); + test("Flaky test", async ({ page }, testInfo) => { await page.goto("https://www.eliostruyf.com", { waitUntil: "domcontentloaded", }); - }); - - test.afterAll(async ({ browser }) => { - await page.close(); - await browser.close(); - }); - test("First test should fail, next should work", async ({}, testInfo) => { if (testInfo.retry === 0) { - expect(true).toBeFalsy(); + await page.evaluate(() => { + const logo: HTMLDivElement | null = + window.document.querySelector(`#logo`); + if (logo) { + logo.style.display = "none"; + } + }); } - expect(true).toBeTruthy(); + + let header = page.locator(`#logo`); + await expect(header).toBeVisible({ timeout: 1000 }); }); test("Skip the test", async () => { @@ -30,4 +36,8 @@ test.describe("Test retry", () => { test("Should work fine", async () => { expect(true).toBeTruthy(); }); + + test("Unexpected", async () => { + test.skip(true, "Don't need to test this."); + }); });