From 29f63373d67c9e456a36ff640ecba7be74cb5344 Mon Sep 17 00:00:00 2001 From: Nicolas Merget <104347736+nmerget@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:01:29 +0200 Subject: [PATCH] chore: improve playwright tests by reducing amount of running projects (#3205) --- showcases/e2e/default.ts | 50 ++++++++++++++----- showcases/e2e/drawer/drawer-a11y.spec.ts | 2 +- showcases/e2e/header/header-a11y.spec.ts | 8 +-- showcases/e2e/home/showcase-home.spec.ts | 6 +-- .../navigation-item-a11y.spec.ts | 5 +- .../e2e/navigation/navigation-a11y.spec.ts | 7 ++- showcases/e2e/popover/popover-a11y.spec.ts | 2 +- showcases/e2e/tooltip/tooltip-a11y.spec.ts | 2 +- showcases/playwright.config.ts | 2 +- 9 files changed, 53 insertions(+), 31 deletions(-) diff --git a/showcases/e2e/default.ts b/showcases/e2e/default.ts index 55ee2697df6..d4e64cfbaf6 100644 --- a/showcases/e2e/default.ts +++ b/showcases/e2e/default.ts @@ -2,6 +2,7 @@ import { expect, type Page, test } from '@playwright/test'; import AxeBuilder from '@axe-core/playwright'; import { close, getCompliance } from 'accessibility-checker'; import { type ICheckerError } from 'accessibility-checker/lib/api/IChecker'; +import { type FullProject } from 'playwright/types/test'; import { COLORS } from './fixtures/variants'; import { setScrollViewport } from './fixtures/viewport'; @@ -19,10 +20,17 @@ export type DefaultSnapshotTestType = { export type DefaultA11yTestType = { axeDisableRules?: string[]; aCheckerDisableRules?: string[]; - skipA11y?: boolean; - preA11y?: (page: Page) => Promise; + skipAxe?: boolean; + preAxe?: (page: Page) => Promise; + preChecker?: (page: Page) => Promise; } & DefaultTestType; +export const hasWebComponentSyntax = (showcase: string): boolean => { + const isAngular = showcase.startsWith('angular'); + const isStencil = showcase.startsWith('stencil'); + return isAngular || isStencil; +}; + export const waitForDBPage = async (page: Page) => { const dbPage = page.locator('.db-page'); // We wait till db-page fully loaded @@ -92,19 +100,27 @@ export const getDefaultScreenshotTest = ({ }); }; +const shouldSkipA11yTest = (project: FullProject): boolean => + project.name === 'firefox' || + project.name === 'webkit' || + project.name.startsWith('mobile'); + export const getA11yTest = ({ path, fixedHeight, axeDisableRules, - skipA11y, - preA11y, - aCheckerDisableRules + skipAxe, + preAxe, + aCheckerDisableRules, + preChecker }: DefaultA11yTestType) => { for (const color of COLORS) { test(`should not have any A11y issues for color ${color}`, async ({ page }, { project }) => { - if (skipA11y) { + const isLevelOne = color.endsWith('-1'); + // We don't need to check color contrast for every project (just for chrome) + if (skipAxe ?? (!isLevelOne && shouldSkipA11yTest(project))) { test.skip(); } @@ -122,8 +138,8 @@ export const getA11yTest = ({ } }, project); - if (preA11y) { - await preA11y(page); + if (preAxe) { + await preAxe(page); } const accessibilityScanResults = await new AxeBuilder({ @@ -138,14 +154,21 @@ export const getA11yTest = ({ } test('test with accessibility checker', async ({ page }, { project }) => { + if (shouldSkipA11yTest(project)) { + // Checking complete DOM in Firefox and Webkit takes very long, we skip this test + // we don't need to check for mobile device - it just changes the viewport + test.skip(); + } + await gotoPage(page, path, 'neutral-bg-basic-level-1', fixedHeight); + + if (preChecker) { + await preChecker(page); + } + let failures: any[] = []; try { - if (project.name === 'firefox') { - // Checking complete DOM in Firefox takes very long, we skip this test for Firefox - test.skip(); - } - + // Makes a call against https://cdn.jsdelivr.net/npm/accessibility-checker-engine const { report } = await getCompliance(page, path); if (isCheckerError(report)) { @@ -159,6 +182,7 @@ export const getA11yTest = ({ } } catch (error) { console.error(error); + failures.push(error); } finally { await close(); } diff --git a/showcases/e2e/drawer/drawer-a11y.spec.ts b/showcases/e2e/drawer/drawer-a11y.spec.ts index a592c525523..4aa5b9638f9 100644 --- a/showcases/e2e/drawer/drawer-a11y.spec.ts +++ b/showcases/e2e/drawer/drawer-a11y.spec.ts @@ -5,7 +5,7 @@ import { getA11yTest } from '../default.ts'; test.describe('DBDrawer', () => { getA11yTest({ path: '01/drawer', - async preA11y(page) { + async preAxe(page) { await page.locator('main').getByRole('button').first().click(); await page.waitForTimeout(1000); } diff --git a/showcases/e2e/header/header-a11y.spec.ts b/showcases/e2e/header/header-a11y.spec.ts index 4aa0f52486b..a66b2c7fdb8 100644 --- a/showcases/e2e/header/header-a11y.spec.ts +++ b/showcases/e2e/header/header-a11y.spec.ts @@ -1,8 +1,10 @@ import { test } from '@playwright/test'; // @ts-expect-error - required for playwright -import { getA11yTest } from '../default.ts'; +import { getA11yTest, hasWebComponentSyntax } from '../default.ts'; test.describe('DBHeader', () => { - const isAngular = process.env.showcase.startsWith('angular'); - getA11yTest({ path: '01/header', skipA11y: isAngular }); + getA11yTest({ + path: '01/header', + skipAxe: hasWebComponentSyntax(process.env.showcase) + }); }); diff --git a/showcases/e2e/home/showcase-home.spec.ts b/showcases/e2e/home/showcase-home.spec.ts index 4b251616f9a..bcec39f3055 100644 --- a/showcases/e2e/home/showcase-home.spec.ts +++ b/showcases/e2e/home/showcase-home.spec.ts @@ -1,6 +1,6 @@ import { expect, test, type Page } from '@playwright/test'; import AxeBuilder from '@axe-core/playwright'; -import { waitForDBPage } from '../default'; +import { hasWebComponentSyntax, waitForDBPage } from '../default'; const testFormComponents = async ( page: Page, @@ -62,10 +62,8 @@ test.describe('Home', () => { }); test('should not have any A11y issues', async ({ page }) => { - const isAngular = process.env.showcase.startsWith('angular'); - // Angular wraps custom components in own tags this will cause a lot of issues with axe-core - if (isAngular) { + if (hasWebComponentSyntax(process.env.showcase)) { test.skip(); } diff --git a/showcases/e2e/navigation-item/navigation-item-a11y.spec.ts b/showcases/e2e/navigation-item/navigation-item-a11y.spec.ts index bd0f78ea65b..22f80f8a925 100644 --- a/showcases/e2e/navigation-item/navigation-item-a11y.spec.ts +++ b/showcases/e2e/navigation-item/navigation-item-a11y.spec.ts @@ -1,13 +1,12 @@ import { test } from '@playwright/test'; // @ts-expect-error - required for playwright -import { getA11yTest } from '../default.ts'; +import { getA11yTest, hasWebComponentSyntax } from '../default.ts'; test.describe('DBNavigationItem', () => { // Set fixed height, because of issues with angulars `ngAfterContentInit` - const isAngular = process.env.showcase.startsWith('angular'); getA11yTest({ path: '05/navigation-item', fixedHeight: 1800, - skipA11y: isAngular + skipAxe: hasWebComponentSyntax(process.env.showcase) }); }); diff --git a/showcases/e2e/navigation/navigation-a11y.spec.ts b/showcases/e2e/navigation/navigation-a11y.spec.ts index 87160dcc2c1..445e53df589 100644 --- a/showcases/e2e/navigation/navigation-a11y.spec.ts +++ b/showcases/e2e/navigation/navigation-a11y.spec.ts @@ -1,11 +1,10 @@ -import { test } from '@playwright/test'; +import { expect, test } from '@playwright/test'; // @ts-expect-error - required for playwright -import { getA11yTest } from '../default.ts'; +import { getA11yTest, hasWebComponentSyntax } from '../default.ts'; test.describe('DBNavigation', () => { - const isAngular = process.env.showcase.startsWith('angular'); getA11yTest({ path: '05/navigation', - skipA11y: isAngular + skipAxe: hasWebComponentSyntax(process.env.showcase) }); }); diff --git a/showcases/e2e/popover/popover-a11y.spec.ts b/showcases/e2e/popover/popover-a11y.spec.ts index 0e0ac53593a..8dd2bee704b 100644 --- a/showcases/e2e/popover/popover-a11y.spec.ts +++ b/showcases/e2e/popover/popover-a11y.spec.ts @@ -8,6 +8,6 @@ const selector = '.db-popover'; test.describe('DBPopover', () => { getA11yTest({ path: '01/popover', - preA11y: async (page) => hoverPre(page, selector) + preAxe: async (page) => hoverPre(page, selector) }); }); diff --git a/showcases/e2e/tooltip/tooltip-a11y.spec.ts b/showcases/e2e/tooltip/tooltip-a11y.spec.ts index 302ed9a95ac..c60af447e01 100644 --- a/showcases/e2e/tooltip/tooltip-a11y.spec.ts +++ b/showcases/e2e/tooltip/tooltip-a11y.spec.ts @@ -8,6 +8,6 @@ const selector = '.db-tooltip'; test.describe('DBTooltip', () => { getA11yTest({ path: '04/tooltip', - preA11y: async (page) => hoverPre(page, selector) + preAxe: async (page) => hoverPre(page, selector) }); }); diff --git a/showcases/playwright.config.ts b/showcases/playwright.config.ts index d630f7ec88f..c569a886fcc 100644 --- a/showcases/playwright.config.ts +++ b/showcases/playwright.config.ts @@ -14,7 +14,7 @@ const config: PlaywrightTestConfig = { '{snapshotDir}/{testFileDir}/showcase/{projectName}/{arg}/{testName}{ext}', snapshotDir: './../__snapshots__', expect: { - timeout: 10_000 + timeout: 30_000 }, /* Run tests in files in parallel */ fullyParallel: true,