From b365051dc24156d810004917efa64dbcbaa64a59 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 2 Jan 2024 16:00:08 +0000 Subject: [PATCH] [Security Solution][Detection Engine] fixes flaky ES|QL tests (#173251) ## Summary - fixes flaky ES|QL tests https://github.com/elastic/kibana/issues/173006 - the reason is an issue in cypress itself https://github.com/cypress-io/cypress/issues/22113. It doesn't have a fix yet, but I tried some suggested workaround, as calling `uncaught:exception` handler after page is opened, so it is registered in the same origin. Which seemed to help, since there was no failures in 300 runs. Usual rate of flakiness before the fix was 3-7 failures per 100 runs. Hopefully, it will work and we won't see any errors in future. ### Checklist Delete any items that are not applicable to this PR. - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - 100 runs for all Detection tests: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4625 - 2 x 100 for just failing ones: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4620, https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4605 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../rule_creation/esql_rule_ess.cy.ts | 41 +++++++++++++++---- .../rule_edit/esql_rule.cy.ts | 9 ++-- .../cypress/tasks/create_new_rule.ts | 9 ++++ 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts index ab0fbb0cf445a..089cfc01ce0bb 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts @@ -20,6 +20,7 @@ import { getDetails, goBackToRulesTable } from '../../../../tasks/rule_details'; import { expectNumberOfRules } from '../../../../tasks/alerts_detection_rules'; import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; import { + expandEsqlQueryBar, fillAboutRuleAndContinue, fillDefineEsqlRuleAndContinue, fillScheduleRuleAndContinue, @@ -34,12 +35,22 @@ import { visit } from '../../../../tasks/navigation'; import { CREATE_RULE_URL } from '../../../../urls/navigation'; +// https://github.com/cypress-io/cypress/issues/22113 +// issue is inside monaco editor, used in ES|QL query input +// calling it after visiting page in each tests, seems fixes the issue +// the only other alternative is patching ResizeObserver, which is something I would like to avoid +const workaroundForResizeObserver = () => + cy.on('uncaught:exception', (err) => { + if (err.message.includes('ResizeObserver loop limit exceeded')) { + return false; + } + }); + describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { const rule = getEsqlRule(); const expectedNumberOfRules = 1; - // FLAKY: https://github.com/elastic/kibana/issues/172618 - describe.skip('creation', () => { + describe('creation', () => { beforeEach(() => { deleteAlertsAndRules(); login(); @@ -47,8 +58,10 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { it('creates an ES|QL rule', function () { visit(CREATE_RULE_URL); + workaroundForResizeObserver(); selectEsqlRuleType(); + expandEsqlQueryBar(); // ensures ES|QL rule in technical preview on create page cy.get(ESQL_TYPE).contains('Technical Preview'); @@ -73,8 +86,10 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { // this test case is important, since field shown in rule override component are coming from ES|QL query, not data view fields API it('creates an ES|QL rule and overrides its name', function () { visit(CREATE_RULE_URL); + workaroundForResizeObserver(); selectEsqlRuleType(); + expandEsqlQueryBar(); fillDefineEsqlRuleAndContinue(rule); fillAboutSpecificEsqlRuleAndContinue({ ...rule, rule_name_override: 'test_id' }); @@ -86,23 +101,26 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { }); }); - // FLAKY: https://github.com/elastic/kibana/issues/172881 - describe.skip('ES|QL query validation', () => { + describe('ES|QL query validation', () => { beforeEach(() => { login(); visit(CREATE_RULE_URL); }); it('shows error when ES|QL query is empty', function () { - selectEsqlRuleType(); + workaroundForResizeObserver(); + selectEsqlRuleType(); + expandEsqlQueryBar(); getDefineContinueButton().click(); cy.get(ESQL_QUERY_BAR).contains('ES|QL query is required'); }); it('proceeds further once invalid query is fixed', function () { - selectEsqlRuleType(); + workaroundForResizeObserver(); + selectEsqlRuleType(); + expandEsqlQueryBar(); getDefineContinueButton().click(); cy.get(ESQL_QUERY_BAR).contains('required'); @@ -115,9 +133,11 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { }); it('shows error when non-aggregating ES|QL query does not [metadata] operator', function () { + workaroundForResizeObserver(); + const invalidNonAggregatingQuery = 'from auditbeat* | limit 5'; selectEsqlRuleType(); - + expandEsqlQueryBar(); fillEsqlQueryBar(invalidNonAggregatingQuery); getDefineContinueButton().click(); @@ -127,11 +147,13 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { }); it('shows error when non-aggregating ES|QL query does not return _id field', function () { + workaroundForResizeObserver(); + const invalidNonAggregatingQuery = 'from auditbeat* [metadata _id, _version, _index] | keep agent.* | limit 5'; selectEsqlRuleType(); - + expandEsqlQueryBar(); fillEsqlQueryBar(invalidNonAggregatingQuery); getDefineContinueButton().click(); @@ -141,12 +163,13 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { }); it('shows error when ES|QL query is invalid', function () { + workaroundForResizeObserver(); const invalidEsqlQuery = 'from auditbeat* [metadata _id, _version, _index] | not_existing_operator'; visit(CREATE_RULE_URL); selectEsqlRuleType(); - + expandEsqlQueryBar(); fillEsqlQueryBar(invalidEsqlQuery); getDefineContinueButton().click(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts index 50980e0add4f8..265263ba495c6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts @@ -9,7 +9,7 @@ import { getEsqlRule } from '../../../../objects/rule'; import { ESQL_QUERY_DETAILS, RULE_NAME_OVERRIDE_DETAILS } from '../../../../screens/rule_details'; -import { ESQL_QUERY_BAR, ESQL_QUERY_BAR_EXPAND_BTN } from '../../../../screens/create_new_rule'; +import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule'; import { createRule } from '../../../../tasks/api_calls/rules'; @@ -18,6 +18,7 @@ import { getDetails } from '../../../../tasks/rule_details'; import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; import { clearEsqlQueryBar, + expandEsqlQueryBar, fillEsqlQueryBar, fillOverrideEsqlRuleName, goToAboutStepTab, @@ -44,8 +45,7 @@ describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => { it('edits ES|QL rule and checks details page', () => { visit(RULES_MANAGEMENT_URL); editFirstRule(); - // expands query bar, so query is not obscured on narrow screens - cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click(); + expandEsqlQueryBar(); // ensure once edit form opened, correct query is displayed in ES|QL input cy.get(ESQL_QUERY_BAR).contains(rule.query); @@ -78,8 +78,7 @@ describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => { visit(RULES_MANAGEMENT_URL); editFirstRule(); - // expands query bar, so query is not obscured on narrow screens - cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click(); + expandEsqlQueryBar(); // ensure once edit form opened, correct query is displayed in ES|QL input cy.get(ESQL_QUERY_BAR).contains(rule.query); diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts index a5f411670ad38..1089834384374 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts @@ -55,6 +55,7 @@ import { EQL_TYPE, ESQL_TYPE, ESQL_QUERY_BAR, + ESQL_QUERY_BAR_EXPAND_BTN, ESQL_QUERY_BAR_INPUT_AREA, FALSE_POSITIVES_INPUT, IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK, @@ -537,6 +538,14 @@ export const fillEsqlQueryBar = (query: string) => { cy.get(ESQL_QUERY_BAR_INPUT_AREA).type(query); }; +/** + * expands query bar, so query is not obscured on narrow screens + * and validation message is not covered by input menu tooltip + */ +export const expandEsqlQueryBar = () => { + cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click(); +}; + export const fillDefineEsqlRuleAndContinue = (rule: EsqlRuleCreateProps) => { cy.get(ESQL_QUERY_BAR).contains('ES|QL query'); fillEsqlQueryBar(rule.query);