[8.11] [ftr] split x-pack accessibility config in 3 groups (#171186)

.buildkite/ftr_configs.yml

@@ -171,7 +171,9 @@ enabled:
   - test/server_integration/http/ssl_with_p12/config.js
   - test/server_integration/http/ssl/config.js
   - test/ui_capabilities/newsfeed_err/config.ts
-  - x-pack/test/accessibility/config.ts
+  - x-pack/test/accessibility/apps/group1/config.ts
+  - x-pack/test/accessibility/apps/group2/config.ts
+  - x-pack/test/accessibility/apps/group3/config.ts
   - x-pack/test/localization/config.ja_jp.ts
   - x-pack/test/localization/config.fr_fr.ts
   - x-pack/test/localization/config.zh_cn.ts

docs/developer/contributing/development-accessibility-tests.asciidoc

@@ -68,7 +68,7 @@ node scripts/functional_test_runner.js --config test/accessibility/config.ts
 -----------
 
 To run the x-pack tests, swap the config file out for
-`x-pack/test/accessibility/config.ts`.
+`x-pack/test/accessibility/apps/{group1,group2,group3}/config.ts`.
 
 The testing is done using https://github.com/dequelabs/axe-core[axe].
 You can run the same thing that runs CI using browser plugins:

docs/developer/contributing/interpreting-ci-failures.asciidoc

@@ -33,7 +33,7 @@ image::images/test_results.png[Buildkite build screenshot]
 Looking at the failure, we first look at the Error and stack trace. In the example below, this test failed to find an element within the timeout;
  `Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj="createSpace"])`
 
-We know the test file from the stack trace was on line 50 of `test/accessibility/apps/spaces.ts` (this test and the stack trace context is kibana/x-pack/ so the file is https://github.com/elastic/kibana/blob/main/x-pack/test/accessibility/apps/spaces.ts#L50).
+We know the test file from the stack trace was on line 50 of `test/accessibility/apps/spaces.ts` (this test and the stack trace context is kibana/x-pack/ so the file is https://github.com/elastic/kibana/blob/main/x-pack/test/accessibility/apps/group1/spaces.ts#L50).
 The function to click on the element was called from a page object method in `test/functional/page_objects/space_selector_page.ts` https://github.com/elastic/kibana/blob/main/x-pack/test/functional/page_objects/space_selector_page.ts#L58
 
 

x-pack/plugins/enterprise_search/README.md

@@ -137,4 +137,4 @@ To track what Cypress is doing while running tests, you can pass in `--config vi
 
 See [our functional test runner README](../../test/functional_enterprise_search).
 
-Our automated accessibility tests can be found in [x-pack/test/accessibility/apps](../../test/accessibility/apps/enterprise_search.ts).
+Our automated accessibility tests can be found in [x-pack/test/accessibility/apps](../../test/accessibility/apps/group3/enterprise_search.ts).

x-pack/plugins/ml/readme.md

@@ -149,7 +149,7 @@ With PATH_TO_CONFIG and other options as follows.
 
     - PATH_TO_CONFIG: `test/accessibility/config.ts`
     - Add `--grep=ml` to the test runner command
-    - Tests are located in `x-pack/test/accessibility/apps`
+    - Tests are located in `x-pack/test/accessibility/apps/group2`
 
 ## Generating docs screenshots
 

x-pack/plugins/transform/readme.md

@@ -151,4 +151,4 @@ With PATH_TO_CONFIG and other options as follows.
          node scripts/functional_tests_server --config test/accessibility/config.ts
          node scripts/functional_test_runner.js --config test/accessibility/config.ts --grep=transform
 
-    Transform accessibility tests are located in `x-pack/test/accessibility/apps`.
+    Transform accessibility tests are located in `x-pack/test/accessibility/apps/group2`.

x-pack/test/accessibility/apps/advanced_settings.ts → x-pack/test/accessibility/apps/group1/advanced_settings.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'settings', 'header']);

x-pack/test/accessibility/apps/group1/config.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import { services } from '../../services';
+import { pageObjects } from '../../page_objects';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(
+    require.resolve('../../../functional/config.base.js')
+  );
+
+  return {
+    ...functionalConfig.getAll(),
+
+    testFiles: [require.resolve('.')],
+
+    pageObjects,
+    services,
+
+    junit: {
+      reportName: 'X-Pack Accessibility Tests - Group 1',
+    },
+  };
+}

x-pack/test/accessibility/apps/dashboard_controls.ts → x-pack/test/accessibility/apps/group1/dashboard_controls.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const a11y = getService('a11y');

x-pack/test/accessibility/apps/dashboard_panel_options.ts → x-pack/test/accessibility/apps/group1/dashboard_panel_options.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
-import { WebElementWrapper } from '../../../../test/functional/services/lib/web_element_wrapper';
+import { FtrProviderContext } from '../../ftr_provider_context';
+import type { WebElementWrapper } from '../../../../../test/functional/services/lib/web_element_wrapper';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const a11y = getService('a11y');

x-pack/test/accessibility/apps/grok_debugger.ts → x-pack/test/accessibility/apps/group1/grok_debugger.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'security']);

x-pack/test/accessibility/apps/home.ts → x-pack/test/accessibility/apps/group1/home.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const { common, home } = getPageObjects(['common', 'home']);

x-pack/test/accessibility/apps/group1/index.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+
+export default ({ loadTestFile }: FtrProviderContext): void => {
+  describe('X-Pack Accessibility Tests - Group 1', function () {
+    loadTestFile(require.resolve('./login_page'));
+    loadTestFile(require.resolve('./kibana_overview'));
+    loadTestFile(require.resolve('./home'));
+    loadTestFile(require.resolve('./management'));
+    loadTestFile(require.resolve('./grok_debugger'));
+    loadTestFile(require.resolve('./search_profiler'));
+    loadTestFile(require.resolve('./painless_lab'));
+    loadTestFile(require.resolve('./uptime'));
+    loadTestFile(require.resolve('./spaces'));
+    loadTestFile(require.resolve('./advanced_settings'));
+    loadTestFile(require.resolve('./dashboard_panel_options'));
+    loadTestFile(require.resolve('./dashboard_controls'));
+    loadTestFile(require.resolve('./users'));
+    loadTestFile(require.resolve('./roles'));
+    loadTestFile(require.resolve('./ingest_node_pipelines'));
+    loadTestFile(require.resolve('./index_lifecycle_management'));
+  });
+};

x-pack/test/accessibility/apps/index_lifecycle_management.ts → x-pack/test/accessibility/apps/group1/index_lifecycle_management.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 const REPO_NAME = 'test';
 const POLICY_NAME = 'ilm-a11y-test';

x-pack/test/accessibility/apps/kibana_overview.ts → x-pack/test/accessibility/apps/group1/kibana_overview.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'home']);

x-pack/test/accessibility/apps/login_page.ts → x-pack/test/accessibility/apps/group1/login_page.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const a11y = getService('a11y');

x-pack/test/accessibility/apps/management.ts → x-pack/test/accessibility/apps/group1/management.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects([

x-pack/test/accessibility/apps/painless_lab.ts → x-pack/test/accessibility/apps/group1/painless_lab.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'security']);

x-pack/test/accessibility/apps/roles.ts → x-pack/test/accessibility/apps/group1/roles.ts

@@ -7,7 +7,7 @@
 
 // a11y tests for spaces, space selection and spacce creation and feature controls
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['security', 'settings']);

x-pack/test/accessibility/apps/search_profiler.ts → x-pack/test/accessibility/apps/group1/search_profiler.ts

@@ -6,7 +6,7 @@
  */
 
 import expect from '@kbn/expect';
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'security']);

x-pack/test/accessibility/apps/spaces.ts → x-pack/test/accessibility/apps/group1/spaces.ts

@@ -7,7 +7,7 @@
 
 // a11y tests for spaces, space selection and space creation and feature controls
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'spaceSelector', 'home', 'header', 'security']);

x-pack/test/accessibility/apps/uptime.ts → x-pack/test/accessibility/apps/group1/uptime.ts

@@ -6,8 +6,8 @@
  */
 
 import moment from 'moment';
-import { FtrProviderContext } from '../ftr_provider_context';
-import { makeChecks } from '../../api_integration/apis/uptime/rest/helper/make_checks';
+import { FtrProviderContext } from '../../ftr_provider_context';
+import { makeChecks } from '../../../api_integration/apis/uptime/rest/helper/make_checks';
 
 const A11Y_TEST_MONITOR_ID = 'a11yTestMonitor';
 
@@ -19,7 +19,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const es = getService('es');
   const toasts = getService('toasts');
 
-  describe('uptime Accessibility', () => {
+  // github.com/elastic/kibana/issues/153601
+  describe.skip('uptime Accessibility', () => {
     before(async () => {
       await esArchiver.load('x-pack/test/functional/es_archives/uptime/blank');
       await makeChecks(es, A11Y_TEST_MONITOR_ID, 150, 1, 1000, {

x-pack/test/accessibility/apps/users.ts → x-pack/test/accessibility/apps/group1/users.ts

@@ -7,7 +7,7 @@
 
 // a11y tests for spaces, space selection and spacce creation and feature controls
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['security', 'settings']);

x-pack/test/accessibility/apps/group2/config.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import { services } from '../../services';
+import { pageObjects } from '../../page_objects';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(
+    require.resolve('../../../functional/config.base.js')
+  );
+
+  return {
+    ...functionalConfig.getAll(),
+
+    testFiles: [require.resolve('.')],
+
+    pageObjects,
+    services,
+
+    junit: {
+      reportName: 'X-Pack Accessibility Tests - Group 2',
+    },
+  };
+}

x-pack/test/accessibility/apps/group2/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+
+export default ({ loadTestFile }: FtrProviderContext): void => {
+  describe('X-Pack Accessibility Tests - Group 2', function () {
+    loadTestFile(require.resolve('./ml'));
+    loadTestFile(require.resolve('./ml_anomaly_detection'));
+    loadTestFile(require.resolve('./transform'));
+    loadTestFile(require.resolve('./lens'));
+  });
+};

x-pack/test/accessibility/apps/lens.ts → x-pack/test/accessibility/apps/group2/lens.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'visualize', 'timePicker', 'home', 'lens']);
@@ -25,11 +25,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     });
 
     after(async () => {
-      await PageObjects.common.navigateToApp('visualize');
-      await listingTable.searchForItemWithName(lensChartName);
-      await listingTable.checkListingSelectAllCheckbox();
-      await listingTable.clickDeleteSelected();
-      await PageObjects.common.clickConfirmOnModal();
       await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional');
       await kibanaServer.importExport.unload(
         'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json'
@@ -173,6 +168,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     it('saves lens chart', async () => {
       await PageObjects.lens.save(lensChartName);
       await a11y.testAppSnapshot();
+      // delete newly created Lens
+      await PageObjects.common.navigateToApp('visualize');
+      await listingTable.searchForItemWithName(lensChartName);
+      await listingTable.checkListingSelectAllCheckbox();
+      await listingTable.clickDeleteSelected();
+      await PageObjects.common.clickConfirmOnModal();
     });
   });
 }

x-pack/test/accessibility/apps/ml.ts → x-pack/test/accessibility/apps/group2/ml.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService }: FtrProviderContext) {
   const a11y = getService('a11y');
@@ -68,7 +68,7 @@ export default function ({ getService }: FtrProviderContext) {
         const dfaClassificationJobTrainingPercent = 30;
 
         const uploadFilePath = require.resolve(
-          '../../functional/apps/ml/data_visualizer/files_to_import/artificial_server_log'
+          '../../../functional/apps/ml/data_visualizer/files_to_import/artificial_server_log'
         );
 
         before(async () => {

x-pack/test/accessibility/apps/ml_anomaly_detection.ts → x-pack/test/accessibility/apps/group2/ml_anomaly_detection.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 interface Detector {
   identifier: string;

x-pack/test/accessibility/apps/transform.ts → x-pack/test/accessibility/apps/group2/transform.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService }: FtrProviderContext) {
   const a11y = getService('a11y');

x-pack/test/accessibility/apps/canvas.ts → x-pack/test/accessibility/apps/group3/canvas.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const a11y = getService('a11y');