From 99a0adf0f71fe731526e25be1cbd4ba552a58cf5 Mon Sep 17 00:00:00 2001 From: Alexi Doak <109488926+doakalexi@users.noreply.github.com> Date: Thu, 2 Nov 2023 07:13:04 -0700 Subject: [PATCH] [ResponseOps] Flaky test on MKI: Alerting APIs Summary actions should schedule actions for summary of alerts on a custom interval (#170268) Resolves https://github.com/elastic/kibana/issues/169204 ## Summary Updates wait for queries to filter on the `ruleId`. To verify the changes used a loop to run all the alerting tests 25x. I also noticed we were using `action` instead of `connector`, so I cleaned up the code. ### To verify - Follow the [docs](https://docs.elastic.dev/serverless/create-project) to create a serverless qa project using the image in this PR - Run the following to verify that the tests pass ``` TEST_CLOUD=1 NODE_TLS_REJECT_UNAUTHORIZED=0 TEST_ES_URL="https://elastic:PASSWORD@ES_URL:443" TEST_KIBANA_URL="https://elastic:PASSWORD@KBN_URL" node --no-warnings scripts/functional_test_runner --config x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts ``` --- .../helpers/alerting_wait_for_helpers.ts | 39 ++++++++++- .../test_suites/common/alerting/rules.ts | 70 +++++++++++-------- .../common/alerting/summary_actions.ts | 43 +++++++----- 3 files changed, 103 insertions(+), 49 deletions(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_wait_for_helpers.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_wait_for_helpers.ts index e69e5303e1d8f..f0a17ca5865aa 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_wait_for_helpers.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_wait_for_helpers.ts @@ -17,15 +17,33 @@ import { runRule } from './alerting_api_helper'; export async function waitForDocumentInIndex({ esClient, indexName, + ruleId, num = 1, }: { esClient: Client; indexName: string; + ruleId: string; num?: number; }): Promise { return await pRetry( async () => { - const response = await esClient.search({ index: indexName, sort: 'date:desc' }); + const response = await esClient.search({ + index: indexName, + sort: 'date:desc', + body: { + query: { + bool: { + must: [ + { + term: { + 'ruleId.keyword': ruleId, + }, + }, + ], + }, + }, + }, + }); if (response.hits.hits.length < num) { throw new Error(`Only found ${response.hits.hits.length} / ${num} documents`); } @@ -38,11 +56,28 @@ export async function waitForDocumentInIndex({ export async function getDocumentsInIndex({ esClient, indexName, + ruleId, }: { esClient: Client; indexName: string; + ruleId: string; }): Promise { - return await esClient.search({ index: indexName }); + return await esClient.search({ + index: indexName, + body: { + query: { + bool: { + must: [ + { + term: { + 'ruleId.keyword': ruleId, + }, + }, + ], + }, + }, + }, + }); } export async function createIndex({ diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts index 3ef5ff6a4785f..b0813a55325f1 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts @@ -41,12 +41,12 @@ export default function ({ getService }: FtrProviderContext) { this.tags(['failsOnMKI']); const RULE_TYPE_ID = '.es-query'; const ALERT_ACTION_INDEX = 'alert-action-es-query'; - let actionId: string; + let connectorId: string; let ruleId: string; afterEach(async () => { await supertest - .delete(`/api/actions/connector/${actionId}`) + .delete(`/api/actions/connector/${connectorId}`) .set('kbn-xsrf', 'foo') .set('x-elastic-internal-origin', 'foo'); await supertest @@ -64,12 +64,12 @@ export default function ({ getService }: FtrProviderContext) { it('should schedule task, run rule and schedule actions when appropriate', async () => { const testStart = new Date(); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -89,7 +89,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -120,6 +120,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(1); @@ -157,12 +158,12 @@ export default function ({ getService }: FtrProviderContext) { it('should pass updated rule params to executor', async () => { const testStart = new Date(); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -182,7 +183,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -213,6 +214,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(1); @@ -253,6 +255,7 @@ export default function ({ getService }: FtrProviderContext) { const resp2 = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, num: 2, }); expect(resp2.hits.hits.length).to.be(2); @@ -276,11 +279,11 @@ export default function ({ getService }: FtrProviderContext) { const testStart = new Date(); // Should fail - const createdAction = await createSlackConnector({ + const createdConnector = await createSlackConnector({ supertest, name: 'Slack Connector: Alerting API test', }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -300,7 +303,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { message: `message: {{rule.id}}`, }, @@ -327,12 +330,12 @@ export default function ({ getService }: FtrProviderContext) { it('should throttle alerts when appropriate', async () => { const testStart = new Date(); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -354,7 +357,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -394,6 +397,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(1); }); @@ -401,12 +405,12 @@ export default function ({ getService }: FtrProviderContext) { it('should throttle alerts with throttled action when appropriate', async () => { const testStart = new Date(); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -427,7 +431,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -472,6 +476,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(1); }); @@ -479,12 +484,12 @@ export default function ({ getService }: FtrProviderContext) { it('should reset throttle window when not firing and should not throttle when changing groups', async () => { const testStart = new Date(); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -505,7 +510,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -530,7 +535,7 @@ export default function ({ getService }: FtrProviderContext) { }, { group: 'recovered', - id: actionId, + id: connectorId, params: { documents: [ { @@ -561,6 +566,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(1); @@ -616,6 +622,7 @@ export default function ({ getService }: FtrProviderContext) { const resp2 = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, num: 2, }); expect(resp2.hits.hits.length).to.be(2); @@ -625,12 +632,12 @@ export default function ({ getService }: FtrProviderContext) { const testStart = new Date(); await createIndex({ esClient, indexName: ALERT_ACTION_INDEX }); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -651,7 +658,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -707,6 +714,7 @@ export default function ({ getService }: FtrProviderContext) { const resp2 = await getDocumentsInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp2.hits.hits.length).to.be(0); }); @@ -715,12 +723,12 @@ export default function ({ getService }: FtrProviderContext) { const testStart = new Date(); await createIndex({ esClient, indexName: ALERT_ACTION_INDEX }); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -741,7 +749,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -798,17 +806,18 @@ export default function ({ getService }: FtrProviderContext) { const resp2 = await getDocumentsInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp2.hits.hits.length).to.be(0); }); it(`should unmute all instances when unmuting an alert`, async () => { - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -829,7 +838,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -881,6 +890,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(1); }); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/summary_actions.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/summary_actions.ts index 107203a61b958..b6eee99e939dc 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/alerting/summary_actions.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/summary_actions.ts @@ -41,13 +41,10 @@ export default function ({ getService }: FtrProviderContext) { const esDeleteAllIndices = getService('esDeleteAllIndices'); describe('Summary actions', function () { - // flaky on MKI, see https://github.com/elastic/kibana/issues/169204 - this.tags(['failsOnMKI']); - const RULE_TYPE_ID = '.es-query'; const ALERT_ACTION_INDEX = 'alert-action-es-query'; const ALERT_INDEX = '.alerts-stack.alerts-default'; - let actionId: string; + let connectorId: string; let ruleId: string; const fields = [ '@timestamp', @@ -67,7 +64,7 @@ export default function ({ getService }: FtrProviderContext) { afterEach(async () => { await supertest - .delete(`/api/actions/connector/${actionId}`) + .delete(`/api/actions/connector/${connectorId}`) .set('kbn-xsrf', 'foo') .set('x-elastic-internal-origin', 'foo') .expect(204); @@ -81,12 +78,12 @@ export default function ({ getService }: FtrProviderContext) { it('should schedule actions for summary of alerts per rule run', async () => { const testStart = new Date(); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -106,7 +103,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -120,6 +117,7 @@ export default function ({ getService }: FtrProviderContext) { recoveredIds: '[{{#alerts.recovered.data}}{{kibana.alert.instance.id}},{{/alerts.recovered.data}}]', date: '{{date}}', + ruleId: '{{rule.id}}', }, ], }, @@ -139,6 +137,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(1); @@ -160,6 +159,7 @@ export default function ({ getService }: FtrProviderContext) { ongoingIds: '[]', recovered: '0', recoveredIds: '[]', + ruleId, }); const alertDocument = resp2.hits.hits[0]._source as Record; @@ -202,12 +202,12 @@ export default function ({ getService }: FtrProviderContext) { it('should filter alerts by kql', async () => { const testStart = new Date(); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -227,7 +227,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -241,6 +241,7 @@ export default function ({ getService }: FtrProviderContext) { recoveredIds: '[{{#alerts.recovered.data}}{{kibana.alert.instance.id}},{{/alerts.recovered.data}}]', date: '{{date}}', + ruleId: '{{rule.id}}', }, ], }, @@ -260,6 +261,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(1); @@ -281,6 +283,7 @@ export default function ({ getService }: FtrProviderContext) { ongoingIds: '[]', recovered: '0', recoveredIds: '[]', + ruleId, }); const alertDocument = resp2.hits.hits[0]._source as Record; @@ -332,12 +335,12 @@ export default function ({ getService }: FtrProviderContext) { await createIndex({ esClient, indexName: ALERT_ACTION_INDEX }); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -358,7 +361,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -372,6 +375,7 @@ export default function ({ getService }: FtrProviderContext) { recoveredIds: '[{{#alerts.recovered.data}}{{kibana.alert.instance.id}},{{/alerts.recovered.data}}]', date: '{{date}}', + ruleId: '{{rule.id}}', }, ], }, @@ -396,18 +400,19 @@ export default function ({ getService }: FtrProviderContext) { const resp = await getDocumentsInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, }); expect(resp.hits.hits.length).to.be(0); }); it('should schedule actions for summary of alerts on a custom interval', async () => { const testStart = new Date(); - const createdAction = await createIndexConnector({ + const createdConnector = await createIndexConnector({ supertest, name: 'Index Connector: Alerting API test', indexName: ALERT_ACTION_INDEX, }); - actionId = createdAction.id; + connectorId = createdConnector.id; const createdRule = await createEsQueryRule({ supertest, @@ -428,7 +433,7 @@ export default function ({ getService }: FtrProviderContext) { actions: [ { group: 'query matched', - id: actionId, + id: connectorId, params: { documents: [ { @@ -442,6 +447,7 @@ export default function ({ getService }: FtrProviderContext) { recoveredIds: '[{{#alerts.recovered.data}}{{kibana.alert.instance.id}},{{/alerts.recovered.data}}]', date: '{{date}}', + ruleId: '{{rule.id}}', }, ], }, @@ -458,6 +464,7 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + ruleId, num: 2, }); @@ -479,6 +486,7 @@ export default function ({ getService }: FtrProviderContext) { ongoingIds: '[]', recovered: '0', recoveredIds: '[]', + ruleId, }); const document1 = resp.hits.hits[0]; @@ -490,6 +498,7 @@ export default function ({ getService }: FtrProviderContext) { ongoingIds: '[query matched,]', recovered: '0', recoveredIds: '[]', + ruleId, }); const alertDocument = resp2.hits.hits[0]._source as Record;