Skip to content

Commit

Permalink
[ftr] split x-pack accessibility config in 3 groups (#171186)
Browse files Browse the repository at this point in the history
## Summary

Splitting long running FTR config:

<img width="1573" alt="image"
src="https://github.com/elastic/kibana/assets/10977896/160b97b4-6468-4c2f-9827-17f6bb9a742a">



After split:

| Config Path | Runtime |
| ------------- | ------------- |
| x-pack/test/accessibility/apps/group1/config.ts |  10m 15s  |
| x-pack/test/accessibility/apps/group2/config.ts | 14m 31s |
| x-pack/test/accessibility/apps/group3/config.ts | 11m 30s  |

---------

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
dmlemeshko and kibanamachine authored Nov 28, 2023
1 parent 085878c commit 0c49603
Show file tree
Hide file tree
Showing 55 changed files with 272 additions and 132 deletions.
4 changes: 3 additions & 1 deletion .buildkite/ftr_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,9 @@ enabled:
- test/server_integration/http/ssl_with_p12/config.js
- test/server_integration/http/ssl/config.js
- test/ui_capabilities/newsfeed_err/config.ts
- x-pack/test/accessibility/config.ts
- x-pack/test/accessibility/apps/group1/config.ts
- x-pack/test/accessibility/apps/group2/config.ts
- x-pack/test/accessibility/apps/group3/config.ts
- x-pack/test/localization/config.ja_jp.ts
- x-pack/test/localization/config.fr_fr.ts
- x-pack/test/localization/config.zh_cn.ts
Expand Down
16 changes: 8 additions & 8 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -872,7 +872,7 @@ packages/kbn-zod-helpers @elastic/security-detection-rule-management
/test/functional/apps/management/ccs_compatibility/_data_views_ccs.ts @elastic/kibana-data-discovery
/test/functional/apps/management/data_views @elastic/kibana-data-discovery
/test/plugin_functional/test_suites/data_plugin @elastic/kibana-data-discovery
/x-pack/test/accessibility/apps/search_sessions.ts @elastic/kibana-data-discovery
/x-pack/test/accessibility/apps/group3/search_sessions.ts @elastic/kibana-data-discovery
/x-pack/test/api_integration/apis/management/rollup/index_patterns_extensions.js @elastic/kibana-data-discovery
/x-pack/test/api_integration/apis/search @elastic/kibana-data-discovery
/x-pack/test/examples/search_examples @elastic/kibana-data-discovery
Expand Down Expand Up @@ -1075,8 +1075,8 @@ x-pack/plugins/infra/server/lib/alerting @elastic/obs-ux-management-team
#CC# /src/plugins/kibana_react/public/code_editor/ @elastic/kibana-presentation

# Machine Learning
/x-pack/test/accessibility/apps/ml.ts @elastic/ml-ui
/x-pack/test/accessibility/apps/ml_embeddables_in_dashboard.ts @elastic/ml-ui
/x-pack/test/accessibility/apps/group2/ml.ts @elastic/ml-ui
/x-pack/test/accessibility/apps/group3/ml_embeddables_in_dashboard.ts @elastic/ml-ui
/x-pack/test/api_integration/apis/ml/ @elastic/ml-ui
/x-pack/test/api_integration_basic/apis/ml/ @elastic/ml-ui
/x-pack/test/functional/apps/ml/ @elastic/ml-ui
Expand All @@ -1090,7 +1090,7 @@ x-pack/plugins/infra/server/lib/alerting @elastic/obs-ux-management-team
/x-pack/test/screenshot_creation/services/ml_screenshots.ts @elastic/ml-ui

# Additional plugins and packages maintained by the ML team.
/x-pack/test/accessibility/apps/transform.ts @elastic/ml-ui
/x-pack/test/accessibility/apps/group2/transform.ts @elastic/ml-ui
/x-pack/test/api_integration/apis/aiops/ @elastic/ml-ui
/x-pack/test/api_integration/apis/transform/ @elastic/ml-ui
/x-pack/test/api_integration_basic/apis/transform/ @elastic/ml-ui
Expand Down Expand Up @@ -1177,10 +1177,10 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
/test/interactive_setup_api_integration/ @elastic/kibana-security
/test/interactive_setup_functional/ @elastic/kibana-security
/test/plugin_functional/test_suites/core_plugins/rendering.ts @elastic/kibana-security
/x-pack/test/accessibility/apps/login_page.ts @elastic/kibana-security
/x-pack/test/accessibility/apps/roles.ts @elastic/kibana-security
/x-pack/test/accessibility/apps/spaces.ts @elastic/kibana-security
/x-pack/test/accessibility/apps/users.ts @elastic/kibana-security
/x-pack/test/accessibility/apps/group1/login_page.ts @elastic/kibana-security
/x-pack/test/accessibility/apps/group1/roles.ts @elastic/kibana-security
/x-pack/test/accessibility/apps/group1/spaces.ts @elastic/kibana-security
/x-pack/test/accessibility/apps/group1/users.ts @elastic/kibana-security
/x-pack/test/api_integration/apis/security/ @elastic/kibana-security
/x-pack/test/api_integration/apis/spaces/ @elastic/kibana-security
/x-pack/test/ui_capabilities/ @elastic/kibana-security
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ node scripts/functional_test_runner.js --config test/accessibility/config.ts
-----------

To run the x-pack tests, swap the config file out for
`x-pack/test/accessibility/config.ts`.
`x-pack/test/accessibility/apps/{group1,group2,group3}/config.ts`.

The testing is done using https://github.com/dequelabs/axe-core[axe].
You can run the same thing that runs CI using browser plugins:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ image::images/test_results.png[Buildkite build screenshot]
Looking at the failure, we first look at the Error and stack trace. In the example below, this test failed to find an element within the timeout;
`Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj="createSpace"])`

We know the test file from the stack trace was on line 50 of `test/accessibility/apps/spaces.ts` (this test and the stack trace context is kibana/x-pack/ so the file is https://github.com/elastic/kibana/blob/main/x-pack/test/accessibility/apps/spaces.ts#L50).
We know the test file from the stack trace was on line 50 of `test/accessibility/apps/spaces.ts` (this test and the stack trace context is kibana/x-pack/ so the file is https://github.com/elastic/kibana/blob/main/x-pack/test/accessibility/apps/group1/spaces.ts#L50).
The function to click on the element was called from a page object method in `test/functional/page_objects/space_selector_page.ts` https://github.com/elastic/kibana/blob/main/x-pack/test/functional/page_objects/space_selector_page.ts#L58


Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugins/enterprise_search/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,4 +137,4 @@ To track what Cypress is doing while running tests, you can pass in `--config vi

See [our functional test runner README](../../test/functional_enterprise_search).

Our automated accessibility tests can be found in [x-pack/test/accessibility/apps](../../test/accessibility/apps/enterprise_search.ts).
Our automated accessibility tests can be found in [x-pack/test/accessibility/apps](../../test/accessibility/apps/group3/enterprise_search.ts).
2 changes: 1 addition & 1 deletion x-pack/plugins/ml/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ With PATH_TO_CONFIG and other options as follows.

- PATH_TO_CONFIG: `test/accessibility/config.ts`
- Add `--grep=ml` to the test runner command
- Tests are located in `x-pack/test/accessibility/apps`
- Tests are located in `x-pack/test/accessibility/apps/group2`

## Generating docs screenshots

Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugins/transform/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,4 +151,4 @@ With PATH_TO_CONFIG and other options as follows.
node scripts/functional_tests_server --config test/accessibility/config.ts
node scripts/functional_test_runner.js --config test/accessibility/config.ts --grep=transform

Transform accessibility tests are located in `x-pack/test/accessibility/apps`.
Transform accessibility tests are located in `x-pack/test/accessibility/apps/group2`.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'settings', 'header']);
Expand Down
29 changes: 29 additions & 0 deletions x-pack/test/accessibility/apps/group1/config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrConfigProviderContext } from '@kbn/test';
import { services } from '../../services';
import { pageObjects } from '../../page_objects';

export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(
require.resolve('../../../functional/config.base.js')
);

return {
...functionalConfig.getAll(),

testFiles: [require.resolve('.')],

pageObjects,
services,

junit: {
reportName: 'X-Pack Accessibility Tests - Group 1',
},
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const a11y = getService('a11y');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { WebElementWrapper } from '../../../../test/functional/services/lib/web_element_wrapper';
import { FtrProviderContext } from '../../ftr_provider_context';
import type { WebElementWrapper } from '../../../../../test/functional/services/lib/web_element_wrapper';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const a11y = getService('a11y');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'security']);
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const { common, home } = getPageObjects(['common', 'home']);
Expand Down
29 changes: 29 additions & 0 deletions x-pack/test/accessibility/apps/group1/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrProviderContext } from '../../../common/ftr_provider_context';

export default ({ loadTestFile }: FtrProviderContext): void => {
describe('X-Pack Accessibility Tests - Group 1', function () {
loadTestFile(require.resolve('./login_page'));
loadTestFile(require.resolve('./kibana_overview'));
loadTestFile(require.resolve('./home'));
loadTestFile(require.resolve('./management'));
loadTestFile(require.resolve('./grok_debugger'));
loadTestFile(require.resolve('./search_profiler'));
loadTestFile(require.resolve('./painless_lab'));
loadTestFile(require.resolve('./uptime'));
loadTestFile(require.resolve('./spaces'));
loadTestFile(require.resolve('./advanced_settings'));
loadTestFile(require.resolve('./dashboard_panel_options'));
loadTestFile(require.resolve('./dashboard_controls'));
loadTestFile(require.resolve('./users'));
loadTestFile(require.resolve('./roles'));
loadTestFile(require.resolve('./ingest_node_pipelines'));
loadTestFile(require.resolve('./index_lifecycle_management'));
});
};
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

const REPO_NAME = 'test';
const POLICY_NAME = 'ilm-a11y-test';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'home']);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const a11y = getService('a11y');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects([
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'security']);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

// a11y tests for spaces, space selection and spacce creation and feature controls

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['security', 'settings']);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*/

import expect from '@kbn/expect';
import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'security']);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

// a11y tests for spaces, space selection and space creation and feature controls

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'spaceSelector', 'home', 'header', 'security']);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
*/

import moment from 'moment';
import { FtrProviderContext } from '../ftr_provider_context';
import { makeChecks } from '../../api_integration/apis/uptime/rest/helper/make_checks';
import { FtrProviderContext } from '../../ftr_provider_context';
import { makeChecks } from '../../../api_integration/apis/uptime/rest/helper/make_checks';

const A11Y_TEST_MONITOR_ID = 'a11yTestMonitor';

Expand All @@ -19,7 +19,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const es = getService('es');
const toasts = getService('toasts');

describe('uptime Accessibility', () => {
// github.com/elastic/kibana/issues/153601
describe.skip('uptime Accessibility', () => {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/uptime/blank');
await makeChecks(es, A11Y_TEST_MONITOR_ID, 150, 1, 1000, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

// a11y tests for spaces, space selection and spacce creation and feature controls

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['security', 'settings']);
Expand Down
29 changes: 29 additions & 0 deletions x-pack/test/accessibility/apps/group2/config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrConfigProviderContext } from '@kbn/test';
import { services } from '../../services';
import { pageObjects } from '../../page_objects';

export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(
require.resolve('../../../functional/config.base.js')
);

return {
...functionalConfig.getAll(),

testFiles: [require.resolve('.')],

pageObjects,
services,

junit: {
reportName: 'X-Pack Accessibility Tests - Group 2',
},
};
}
17 changes: 17 additions & 0 deletions x-pack/test/accessibility/apps/group2/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrProviderContext } from '../../../common/ftr_provider_context';

export default ({ loadTestFile }: FtrProviderContext): void => {
describe('X-Pack Accessibility Tests - Group 2', function () {
loadTestFile(require.resolve('./ml'));
loadTestFile(require.resolve('./ml_anomaly_detection'));
loadTestFile(require.resolve('./transform'));
loadTestFile(require.resolve('./lens'));
});
};
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'visualize', 'timePicker', 'home', 'lens']);
Expand All @@ -25,11 +25,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});

after(async () => {
await PageObjects.common.navigateToApp('visualize');
await listingTable.searchForItemWithName(lensChartName);
await listingTable.checkListingSelectAllCheckbox();
await listingTable.clickDeleteSelected();
await PageObjects.common.clickConfirmOnModal();
await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional');
await kibanaServer.importExport.unload(
'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json'
Expand Down Expand Up @@ -173,6 +168,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
it('saves lens chart', async () => {
await PageObjects.lens.save(lensChartName);
await a11y.testAppSnapshot();
// delete newly created Lens
await PageObjects.common.navigateToApp('visualize');
await listingTable.searchForItemWithName(lensChartName);
await listingTable.checkListingSelectAllCheckbox();
await listingTable.clickDeleteSelected();
await PageObjects.common.clickConfirmOnModal();
});
});
}
Loading

0 comments on commit 0c49603

Please sign in to comment.