A11y: Create a11y test provider and revamp a11y addon

code/.storybook/preview.tsx

@@ -357,6 +357,9 @@ export const parameters = {
       opacity: 0.4,
     },
   },
+  a11y: {
+    warnings: ['minor', 'moderate', 'serious', 'critical'],
+  },
 };
 
 export const tags = ['test', 'vitest'];

code/.storybook/storybook.setup.ts

@@ -6,6 +6,8 @@ import { userEvent as storybookEvent, expect as storybookExpect } from '@storybo
 // eslint-disable-next-line import/namespace
 import * as testAnnotations from '@storybook/experimental-addon-test/preview';
 
+import * as a11yAddonAnnotations from '@storybook/addon-a11y/preview';
+
 import * as coreAnnotations from '../addons/toolbars/template/stories/preview';
 import * as componentAnnotations from '../core/template/stories/preview';
 // register global components used in many stories
@@ -15,10 +17,9 @@ import * as projectAnnotations from './preview';
 vi.spyOn(console, 'warn').mockImplementation((...args) => console.log(...args));
 
 const annotations = setProjectAnnotations([
-  // @ts-expect-error check type errors later
   projectAnnotations,
-  // @ts-expect-error check type errors later
   componentAnnotations,
+  a11yAddonAnnotations,
   coreAnnotations,
   testAnnotations,
   {

code/addons/a11y/README.md

@@ -161,9 +161,9 @@ MyNonCheckedStory.parameters = {
 };
 ```
 
-## Parameters
+## Parameters and globals
 
-For more customizability use parameters to configure [aXe options](https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#api-name-axeconfigure).
+For more customizability use parameters and globals to configure [aXe options](https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#api-name-axeconfigure).
 You can override these options [at story level too](https://storybook.js.org/docs/react/configure/features-and-behavior#per-story-options).
 
 ```js
@@ -180,10 +180,16 @@ export default {
       config: {},
       // axe-core optionsParameter (https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#options-parameter)
       options: {},
-      // optional flag to prevent the automatic check
-      manual: true,
+      // Defines which impact levels will be considered as warnings instead of errors if executed via Storybook's component testing
+      warnings: <'minor' | 'moderate' | 'serious' | 'critical'>[]
     },
   },
+  globals: {
+    a11y: {
+      // optional flag to prevent the automatic check
+      manual: true,
+    }
+  }
 };
 
 export const accessible = () => <button>Accessible button</button>;
@@ -193,6 +199,27 @@ export const inaccessible = () => (
 );
 ```
 
+## Violation impact levels
+
+By default, the addon will consider all violations as errors. However, you can configure the addon to consider some violations as warnings instead of errors. This can be useful when @storybook/addon-a11y is used in combination with `@storybook/experimental-addon-test`. To do this, set the `warnings` parameter in the `a11y` object to an array of impact levels that should be considered as warnings.
+
+```js
+export default {
+  title: 'button',
+  parameters: {
+    a11y: {
+      /**
+       * @default [ ]
+       * @type: Array<'minor' | 'moderate' | 'serious' | 'critical'>
+       */
+      warnings: ['minor', 'moderate'],
+    },
+  },
+};
+```
+
+In this example, all violations with an impact level of `minor` or `moderate` will be considered as warnings. All other violations will be considered as errors. When running automated UI tests featured by Vitest, all violations with an impact level of `serious` or `critical` will now fail the test. This failure is reflected as an error in the sidebar or when running Vitest separately. `minor` and `moderate` violations will be reported as warnings but will not fail the test.
+
 ## Automate accessibility tests with test runner
 
 The test runner does not apply any rules that you have set on your stories by default. You can configure the runner to correctly apply the rules by [following the guide on the Storybook docs](https://storybook.js.org/docs/writing-tests/accessibility-testing#automate-accessibility-tests-with-test-runner).

code/addons/a11y/package.json

@@ -56,12 +56,16 @@
   },
   "dependencies": {
     "@storybook/addon-highlight": "workspace:*",
-    "axe-core": "^4.2.0"
+    "@storybook/test": "workspace:*",
+    "axe-core": "^4.2.0",
+    "vitest-axe": "^0.1.0"
   },
   "devDependencies": {
     "@storybook/global": "^5.0.0",
     "@storybook/icons": "^1.2.12",
     "@testing-library/react": "^14.0.0",
+    "picocolors": "^1.1.0",
+    "pretty-format": "29.7.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-resize-detector": "^7.1.2",

code/addons/a11y/src/a11yRunner.test.ts

@@ -22,7 +22,6 @@ describe('a11yRunner', () => {
     await import('./a11yRunner');
 
     expect(mockedAddons.getChannel).toHaveBeenCalled();
-    expect(mockChannel.on).toHaveBeenCalledWith(EVENTS.REQUEST, expect.any(Function));
     expect(mockChannel.on).toHaveBeenCalledWith(EVENTS.MANUAL, expect.any(Function));
   });
 });

code/addons/a11y/src/a11yRunner.ts

@@ -8,64 +8,36 @@ import type { A11yParameters } from './params';
 const { document } = global;
 
 const channel = addons.getChannel();
-// Holds axe core running state
-let active = false;
-// Holds latest story we requested a run
-let activeStoryId: string | undefined;
 
 const defaultParameters = { config: {}, options: {} };
 
-/** Handle A11yContext events. Because the event are sent without manual check, we split calls */
-const handleRequest = async (storyId: string, input: A11yParameters | null) => {
-  if (!input?.manual) {
-    await run(storyId, input ?? defaultParameters);
-  }
-};
-
-const run = async (storyId: string, input: A11yParameters = defaultParameters) => {
-  activeStoryId = storyId;
-  try {
-    if (!active) {
-      active = true;
-      channel.emit(EVENTS.RUNNING);
-      const { default: axe } = await import('axe-core');
-
-      const { element = '#storybook-root', config, options = {} } = input;
-      const htmlElement = document.querySelector(element as string);
+export const run = async (input: A11yParameters = defaultParameters) => {
+  const { default: axe } = await import('axe-core');
 
-      if (!htmlElement) {
-        return;
-      }
+  const { element = '#storybook-root', config, options = {} } = input;
+  const htmlElement = document.querySelector(element as string) ?? document.body;
 
-      axe.reset();
-      if (config) {
-        axe.configure(config);
-      }
+  if (!htmlElement) {
+    return;
+  }
 
-      const result = await axe.run(htmlElement, options);
+  axe.reset();
+  if (config) {
+    axe.configure(config);
+  }
 
-      // Axe result contains class instances, which telejson deserializes in a
-      // way that violates:
-      //  Content Security Policy directive: "script-src 'self' 'unsafe-inline'".
-      const resultJson = JSON.parse(JSON.stringify(result));
+  return axe.run(htmlElement, options);
+};
 
-      // It's possible that we requested a new run on a different story.
-      // Unfortunately, axe doesn't support a cancel method to abort current run.
-      // We check if the story we run against is still the current one,
-      // if not, trigger a new run using the current story
-      if (activeStoryId === storyId) {
-        channel.emit(EVENTS.RESULT, resultJson);
-      } else {
-        active = false;
-        run(activeStoryId);
-      }
-    }
+channel.on(EVENTS.MANUAL, async (storyId: string, input: A11yParameters = defaultParameters) => {
+  try {
+    const result = await run(input);
+    // Axe result contains class instances, which telejson deserializes in a
+    // way that violates:
+    //  Content Security Policy directive: "script-src 'self' 'unsafe-inline'".
+    const resultJson = JSON.parse(JSON.stringify(result));
+    channel.emit(EVENTS.RESULT, resultJson, storyId);
   } catch (error) {
     channel.emit(EVENTS.ERROR, error);
-  } finally {
-    active = false;
   }
-};
-
-channel.on(EVENTS.REQUEST, handleRequest);
-channel.on(EVENTS.MANUAL, run);
+});