chore: improve playwright tests by reducing amount of running projects (

#3205)

showcases/e2e/default.ts

@@ -2,6 +2,7 @@ import { expect, type Page, test } from '@playwright/test';
 import AxeBuilder from '@axe-core/playwright';
 import { close, getCompliance } from 'accessibility-checker';
 import { type ICheckerError } from 'accessibility-checker/lib/api/IChecker';
+import { type FullProject } from 'playwright/types/test';
 import { COLORS } from './fixtures/variants';
 import { setScrollViewport } from './fixtures/viewport';
 
@@ -19,10 +20,17 @@ export type DefaultSnapshotTestType = {
 export type DefaultA11yTestType = {
 	axeDisableRules?: string[];
 	aCheckerDisableRules?: string[];
-	skipA11y?: boolean;
-	preA11y?: (page: Page) => Promise<void>;
+	skipAxe?: boolean;
+	preAxe?: (page: Page) => Promise<void>;
+	preChecker?: (page: Page) => Promise<void>;
 } & DefaultTestType;
 
+export const hasWebComponentSyntax = (showcase: string): boolean => {
+	const isAngular = showcase.startsWith('angular');
+	const isStencil = showcase.startsWith('stencil');
+	return isAngular || isStencil;
+};
+
 export const waitForDBPage = async (page: Page) => {
 	const dbPage = page.locator('.db-page');
 	// We wait till db-page fully loaded
@@ -92,19 +100,27 @@ export const getDefaultScreenshotTest = ({
 	});
 };
 
+const shouldSkipA11yTest = (project: FullProject): boolean =>
+	project.name === 'firefox' ||
+	project.name === 'webkit' ||
+	project.name.startsWith('mobile');
+
 export const getA11yTest = ({
 	path,
 	fixedHeight,
 	axeDisableRules,
-	skipA11y,
-	preA11y,
-	aCheckerDisableRules
+	skipAxe,
+	preAxe,
+	aCheckerDisableRules,
+	preChecker
 }: DefaultA11yTestType) => {
 	for (const color of COLORS) {
 		test(`should not have any A11y issues for color ${color}`, async ({
 			page
 		}, { project }) => {
-			if (skipA11y) {
+			const isLevelOne = color.endsWith('-1');
+			// We don't need to check color contrast for every project (just for chrome)
+			if (skipAxe ?? (!isLevelOne && shouldSkipA11yTest(project))) {
 				test.skip();
 			}
 
@@ -122,8 +138,8 @@ export const getA11yTest = ({
 				}
 			}, project);
 
-			if (preA11y) {
-				await preA11y(page);
+			if (preAxe) {
+				await preAxe(page);
 			}
 
 			const accessibilityScanResults = await new AxeBuilder({
@@ -138,14 +154,21 @@ export const getA11yTest = ({
 	}
 
 	test('test with accessibility checker', async ({ page }, { project }) => {
+		if (shouldSkipA11yTest(project)) {
+			// Checking complete DOM in Firefox and Webkit takes very long, we skip this test
+			// we don't need to check for mobile device - it just changes the viewport
+			test.skip();
+		}
+
 		await gotoPage(page, path, 'neutral-bg-basic-level-1', fixedHeight);
+
+		if (preChecker) {
+			await preChecker(page);
+		}
+
 		let failures: any[] = [];
 		try {
-			if (project.name === 'firefox') {
-				// Checking complete DOM in Firefox takes very long, we skip this test for Firefox
-				test.skip();
-			}
-
+			// Makes a call against https://cdn.jsdelivr.net/npm/accessibility-checker-engine
 			const { report } = await getCompliance(page, path);
 
 			if (isCheckerError(report)) {
@@ -159,6 +182,7 @@ export const getA11yTest = ({
 			}
 		} catch (error) {
 			console.error(error);
+			failures.push(error);
 		} finally {
 			await close();
 		}

showcases/e2e/drawer/drawer-a11y.spec.ts

@@ -5,7 +5,7 @@ import { getA11yTest } from '../default.ts';
 test.describe('DBDrawer', () => {
 	getA11yTest({
 		path: '01/drawer',
-		async preA11y(page) {
+		async preAxe(page) {
 			await page.locator('main').getByRole('button').first().click();
 			await page.waitForTimeout(1000);
 		}

showcases/e2e/header/header-a11y.spec.ts

@@ -1,8 +1,10 @@
 import { test } from '@playwright/test';
 // @ts-expect-error - required for playwright
-import { getA11yTest } from '../default.ts';
+import { getA11yTest, hasWebComponentSyntax } from '../default.ts';
 
 test.describe('DBHeader', () => {
-	const isAngular = process.env.showcase.startsWith('angular');
-	getA11yTest({ path: '01/header', skipA11y: isAngular });
+	getA11yTest({
+		path: '01/header',
+		skipAxe: hasWebComponentSyntax(process.env.showcase)
+	});
 });

showcases/e2e/home/showcase-home.spec.ts

@@ -1,6 +1,6 @@
 import { expect, test, type Page } from '@playwright/test';
 import AxeBuilder from '@axe-core/playwright';
-import { waitForDBPage } from '../default';
+import { hasWebComponentSyntax, waitForDBPage } from '../default';
 
 const testFormComponents = async (
 	page: Page,
@@ -62,10 +62,8 @@ test.describe('Home', () => {
 	});
 
 	test('should not have any A11y issues', async ({ page }) => {
-		const isAngular = process.env.showcase.startsWith('angular');
-
 		// Angular wraps custom components in own tags this will cause a lot of issues with axe-core
-		if (isAngular) {
+		if (hasWebComponentSyntax(process.env.showcase)) {
 			test.skip();
 		}
 

showcases/e2e/navigation-item/navigation-item-a11y.spec.ts

@@ -1,13 +1,12 @@
 import { test } from '@playwright/test';
 // @ts-expect-error - required for playwright
-import { getA11yTest } from '../default.ts';
+import { getA11yTest, hasWebComponentSyntax } from '../default.ts';
 
 test.describe('DBNavigationItem', () => {
 	// Set fixed height, because of issues with angulars `ngAfterContentInit`
-	const isAngular = process.env.showcase.startsWith('angular');
 	getA11yTest({
 		path: '05/navigation-item',
 		fixedHeight: 1800,
-		skipA11y: isAngular
+		skipAxe: hasWebComponentSyntax(process.env.showcase)
 	});
 });

showcases/e2e/navigation/navigation-a11y.spec.ts

@@ -1,11 +1,10 @@
-import { test } from '@playwright/test';
+import { expect, test } from '@playwright/test';
 // @ts-expect-error - required for playwright
-import { getA11yTest } from '../default.ts';
+import { getA11yTest, hasWebComponentSyntax } from '../default.ts';
 
 test.describe('DBNavigation', () => {
-	const isAngular = process.env.showcase.startsWith('angular');
 	getA11yTest({
 		path: '05/navigation',
-		skipA11y: isAngular
+		skipAxe: hasWebComponentSyntax(process.env.showcase)
 	});
 });

showcases/e2e/popover/popover-a11y.spec.ts

@@ -8,6 +8,6 @@ const selector = '.db-popover';
 test.describe('DBPopover', () => {
 	getA11yTest({
 		path: '01/popover',
-		preA11y: async (page) => hoverPre(page, selector)
+		preAxe: async (page) => hoverPre(page, selector)
 	});
 });

showcases/e2e/tooltip/tooltip-a11y.spec.ts

@@ -8,6 +8,6 @@ const selector = '.db-tooltip';
 test.describe('DBTooltip', () => {
 	getA11yTest({
 		path: '04/tooltip',
-		preA11y: async (page) => hoverPre(page, selector)
+		preAxe: async (page) => hoverPre(page, selector)
 	});
 });

showcases/playwright.config.ts

@@ -14,7 +14,7 @@ const config: PlaywrightTestConfig = {
 		'{snapshotDir}/{testFileDir}/showcase/{projectName}/{arg}/{testName}{ext}',
 	snapshotDir: './../__snapshots__',
 	expect: {
-		timeout: 10_000
+		timeout: 30_000
 	},
 	/* Run tests in files in parallel */
 	fullyParallel: true,