Skip to content

Commit

Permalink
[Security Solution][Detection Engine] fixes flaky ES|QL tests (#173251)
Browse files Browse the repository at this point in the history
## Summary

- fixes flaky ES|QL tests
#173006
- the reason is an issue in cypress itself
cypress-io/cypress#22113. It doesn't have a
fix yet, but I tried some suggested workaround, as calling
`uncaught:exception` handler after page is opened, so it is registered
in the same origin. Which seemed to help, since there was no failures in
300 runs. Usual rate of flakiness before the fix was 3-7 failures per
100 runs. Hopefully, it will work and we won't see any errors in future.


### Checklist

Delete any items that are not applicable to this PR.

- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- 100 runs for all Detection tests:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4625
- 2 x 100 for just failing ones:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4620,
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4605

---------

Co-authored-by: kibanamachine <[email protected]>
  • Loading branch information
vitaliidm and kibanamachine authored Jan 2, 2024
1 parent 04b832b commit b365051
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { getDetails, goBackToRulesTable } from '../../../../tasks/rule_details';
import { expectNumberOfRules } from '../../../../tasks/alerts_detection_rules';
import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common';
import {
expandEsqlQueryBar,
fillAboutRuleAndContinue,
fillDefineEsqlRuleAndContinue,
fillScheduleRuleAndContinue,
Expand All @@ -34,21 +35,33 @@ import { visit } from '../../../../tasks/navigation';

import { CREATE_RULE_URL } from '../../../../urls/navigation';

// https://github.com/cypress-io/cypress/issues/22113
// issue is inside monaco editor, used in ES|QL query input
// calling it after visiting page in each tests, seems fixes the issue
// the only other alternative is patching ResizeObserver, which is something I would like to avoid
const workaroundForResizeObserver = () =>
cy.on('uncaught:exception', (err) => {
if (err.message.includes('ResizeObserver loop limit exceeded')) {
return false;
}
});

describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
const rule = getEsqlRule();
const expectedNumberOfRules = 1;

// FLAKY: https://github.com/elastic/kibana/issues/172618
describe.skip('creation', () => {
describe('creation', () => {
beforeEach(() => {
deleteAlertsAndRules();
login();
});

it('creates an ES|QL rule', function () {
visit(CREATE_RULE_URL);
workaroundForResizeObserver();

selectEsqlRuleType();
expandEsqlQueryBar();

// ensures ES|QL rule in technical preview on create page
cy.get(ESQL_TYPE).contains('Technical Preview');
Expand All @@ -73,8 +86,10 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
// this test case is important, since field shown in rule override component are coming from ES|QL query, not data view fields API
it('creates an ES|QL rule and overrides its name', function () {
visit(CREATE_RULE_URL);
workaroundForResizeObserver();

selectEsqlRuleType();
expandEsqlQueryBar();

fillDefineEsqlRuleAndContinue(rule);
fillAboutSpecificEsqlRuleAndContinue({ ...rule, rule_name_override: 'test_id' });
Expand All @@ -86,23 +101,26 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
});
});

// FLAKY: https://github.com/elastic/kibana/issues/172881
describe.skip('ES|QL query validation', () => {
describe('ES|QL query validation', () => {
beforeEach(() => {
login();
visit(CREATE_RULE_URL);
});
it('shows error when ES|QL query is empty', function () {
selectEsqlRuleType();
workaroundForResizeObserver();

selectEsqlRuleType();
expandEsqlQueryBar();
getDefineContinueButton().click();

cy.get(ESQL_QUERY_BAR).contains('ES|QL query is required');
});

it('proceeds further once invalid query is fixed', function () {
selectEsqlRuleType();
workaroundForResizeObserver();

selectEsqlRuleType();
expandEsqlQueryBar();
getDefineContinueButton().click();

cy.get(ESQL_QUERY_BAR).contains('required');
Expand All @@ -115,9 +133,11 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
});

it('shows error when non-aggregating ES|QL query does not [metadata] operator', function () {
workaroundForResizeObserver();

const invalidNonAggregatingQuery = 'from auditbeat* | limit 5';
selectEsqlRuleType();

expandEsqlQueryBar();
fillEsqlQueryBar(invalidNonAggregatingQuery);
getDefineContinueButton().click();

Expand All @@ -127,11 +147,13 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
});

it('shows error when non-aggregating ES|QL query does not return _id field', function () {
workaroundForResizeObserver();

const invalidNonAggregatingQuery =
'from auditbeat* [metadata _id, _version, _index] | keep agent.* | limit 5';

selectEsqlRuleType();

expandEsqlQueryBar();
fillEsqlQueryBar(invalidNonAggregatingQuery);
getDefineContinueButton().click();

Expand All @@ -141,12 +163,13 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
});

it('shows error when ES|QL query is invalid', function () {
workaroundForResizeObserver();
const invalidEsqlQuery =
'from auditbeat* [metadata _id, _version, _index] | not_existing_operator';
visit(CREATE_RULE_URL);

selectEsqlRuleType();

expandEsqlQueryBar();
fillEsqlQueryBar(invalidEsqlQuery);
getDefineContinueButton().click();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { getEsqlRule } from '../../../../objects/rule';

import { ESQL_QUERY_DETAILS, RULE_NAME_OVERRIDE_DETAILS } from '../../../../screens/rule_details';

import { ESQL_QUERY_BAR, ESQL_QUERY_BAR_EXPAND_BTN } from '../../../../screens/create_new_rule';
import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule';

import { createRule } from '../../../../tasks/api_calls/rules';

Expand All @@ -18,6 +18,7 @@ import { getDetails } from '../../../../tasks/rule_details';
import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common';
import {
clearEsqlQueryBar,
expandEsqlQueryBar,
fillEsqlQueryBar,
fillOverrideEsqlRuleName,
goToAboutStepTab,
Expand All @@ -44,8 +45,7 @@ describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => {
it('edits ES|QL rule and checks details page', () => {
visit(RULES_MANAGEMENT_URL);
editFirstRule();
// expands query bar, so query is not obscured on narrow screens
cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click();
expandEsqlQueryBar();
// ensure once edit form opened, correct query is displayed in ES|QL input
cy.get(ESQL_QUERY_BAR).contains(rule.query);

Expand Down Expand Up @@ -78,8 +78,7 @@ describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => {
visit(RULES_MANAGEMENT_URL);
editFirstRule();

// expands query bar, so query is not obscured on narrow screens
cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click();
expandEsqlQueryBar();
// ensure once edit form opened, correct query is displayed in ES|QL input
cy.get(ESQL_QUERY_BAR).contains(rule.query);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import {
EQL_TYPE,
ESQL_TYPE,
ESQL_QUERY_BAR,
ESQL_QUERY_BAR_EXPAND_BTN,
ESQL_QUERY_BAR_INPUT_AREA,
FALSE_POSITIVES_INPUT,
IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK,
Expand Down Expand Up @@ -537,6 +538,14 @@ export const fillEsqlQueryBar = (query: string) => {
cy.get(ESQL_QUERY_BAR_INPUT_AREA).type(query);
};

/**
* expands query bar, so query is not obscured on narrow screens
* and validation message is not covered by input menu tooltip
*/
export const expandEsqlQueryBar = () => {
cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click();
};

export const fillDefineEsqlRuleAndContinue = (rule: EsqlRuleCreateProps) => {
cy.get(ESQL_QUERY_BAR).contains('ES|QL query');
fillEsqlQueryBar(rule.query);
Expand Down

0 comments on commit b365051

Please sign in to comment.