From 4a0eefac0e28b6d17e0eed6567ffe3c9e6a17874 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Tue, 15 Oct 2024 13:46:21 -0700 Subject: [PATCH 1/4] Add error handling/retry logic for search source alert tests --- .../discover/search_source_alert.ts | 86 +++++++++--------- .../discover/search_source_alert.ts | 88 ++++++++++--------- 2 files changed, 92 insertions(+), 82 deletions(-) diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 8c176c61530b7..5ab3f83a8fcef 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -45,53 +45,59 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { let connectorId: string; const createSourceIndex = () => - es.index({ - index: SOURCE_DATA_VIEW, - body: { - settings: { number_of_shards: 1 }, - mappings: { - properties: { - '@timestamp': { type: 'date' }, - message: { type: 'keyword' }, - }, + retry.try(() => + createIndex(SOURCE_DATA_VIEW, { + '@timestamp': { type: 'date' }, + message: { type: 'keyword' }, + }) + ); + + const createOutputDataIndex = () => + retry.try(() => + createIndex(OUTPUT_DATA_VIEW, { + rule_id: { type: 'text' }, + rule_name: { type: 'text' }, + alert_id: { type: 'text' }, + context_link: { type: 'text' }, + }) + ); + + async function createIndex(index: string, properties: unknown) { + try { + await es.index({ + index, + body: { + settings: { number_of_shards: 1 }, + mappings: { properties }, }, - }, - }); + }); + } catch (e) { + log.error(`Failed to create index "${index}" with error "${e.message}"`); + } + } - const generateNewDocs = async (docsNumber: number) => { + async function generateNewDocs(docsNumber: number, index = SOURCE_DATA_VIEW) { const mockMessages = Array.from({ length: docsNumber }, (_, i) => `msg-${i}`); const dateNow = new Date(); const dateToSet = new Date(dateNow); dateToSet.setMinutes(dateNow.getMinutes() - 10); - for (const message of mockMessages) { - await es.transport.request({ - path: `/${SOURCE_DATA_VIEW}/_doc`, - method: 'POST', - body: { - '@timestamp': dateToSet.toISOString(), - message, - }, - }); + try { + await Promise.all( + mockMessages.map((message) => + es.transport.request({ + path: `/${index}/_doc`, + method: 'POST', + body: { + '@timestamp': dateToSet.toISOString(), + message, + }, + }) + ) + ); + } catch (e) { + log.error(`Failed to generate new docs in "${index}" with error "${e.message}"`); } - }; - - const createOutputDataIndex = () => - es.index({ - index: OUTPUT_DATA_VIEW, - body: { - settings: { - number_of_shards: 1, - }, - mappings: { - properties: { - rule_id: { type: 'text' }, - rule_name: { type: 'text' }, - alert_id: { type: 'text' }, - context_link: { type: 'text' }, - }, - }, - }, - }); + } const deleteAlerts = (alertIds: string[]) => asyncForEach(alertIds, async (alertId: string) => { diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index 69db169209d59..e7e81925ab2a1 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -48,53 +48,59 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { let connectorId: string; const createSourceIndex = () => - es.index({ - index: SOURCE_DATA_VIEW, - body: { - settings: { number_of_shards: 1 }, - mappings: { - properties: { - '@timestamp': { type: 'date' }, - message: { type: 'keyword' }, - }, + retry.try(() => + createIndex(SOURCE_DATA_VIEW, { + '@timestamp': { type: 'date' }, + message: { type: 'keyword' }, + }) + ); + + const createOutputDataIndex = () => + retry.try(() => + createIndex(OUTPUT_DATA_VIEW, { + rule_id: { type: 'text' }, + rule_name: { type: 'text' }, + alert_id: { type: 'text' }, + context_link: { type: 'text' }, + }) + ); + + async function createIndex(index: string, properties: unknown) { + try { + await es.index({ + index, + body: { + settings: { number_of_shards: 1 }, + mappings: { properties }, }, - }, - }); + }); + } catch (e) { + log.error(`Failed to create index "${index}" with error "${e.message}"`); + } + } - const generateNewDocs = async (docsNumber: number) => { + async function generateNewDocs(docsNumber: number, index = SOURCE_DATA_VIEW) { const mockMessages = Array.from({ length: docsNumber }, (_, i) => `msg-${i}`); const dateNow = new Date(); const dateToSet = new Date(dateNow); dateToSet.setMinutes(dateNow.getMinutes() - 10); - for (const message of mockMessages) { - await es.transport.request({ - path: `/${SOURCE_DATA_VIEW}/_doc`, - method: 'POST', - body: { - '@timestamp': dateToSet.toISOString(), - message, - }, - }); + try { + await Promise.all( + mockMessages.map((message) => + es.transport.request({ + path: `/${index}/_doc`, + method: 'POST', + body: { + '@timestamp': dateToSet.toISOString(), + message, + }, + }) + ) + ); + } catch (e) { + log.error(`Failed to generate new docs in "${index}" with error "${e.message}"`); } - }; - - const createOutputDataIndex = () => - es.index({ - index: OUTPUT_DATA_VIEW, - body: { - settings: { - number_of_shards: 1, - }, - mappings: { - properties: { - rule_id: { type: 'text' }, - rule_name: { type: 'text' }, - alert_id: { type: 'text' }, - context_link: { type: 'text' }, - }, - }, - }, - }); + } const deleteAlerts = (alertIds: string[]) => asyncForEach(alertIds, async (alertId: string) => { @@ -366,8 +372,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }; describe('Search source Alert', function () { - // see details: https://github.com/elastic/kibana/issues/193842 - this.tags(['failsOnMKI']); before(async () => { await security.testUser.setRoles(['discover_alert']); await PageObjects.svlCommonPage.loginAsAdmin(); From cc4367f78b5dfa3283b744e27039f6226d65acf6 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Wed, 20 Nov 2024 15:13:42 -0700 Subject: [PATCH 2/4] Fix data-test-subj for MKI tests --- .../discover/search_source_alert.ts | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index e7e81925ab2a1..62c6650e9db75 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -506,7 +506,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.click('thresholdPopover'); await testSubjects.setValue('alertThresholdInput0', '1'); - await testSubjects.click('saveEditedRuleButton'); + + // Different save buttons in serverless + if (await testSubjects.exists('saveEditedRuleButton')) { + await testSubjects.click('saveEditedRuleButton'); + } else { + await testSubjects.click('rulePageFooterSaveButton'); + } await PageObjects.header.waitUntilLoadingHasFinished(); await openAlertResults(RULE_NAME); @@ -656,8 +662,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await retry.waitFor('rule name value is correct', async () => { - await testSubjects.setValue('ruleNameInput', newAlert); - const ruleName = await testSubjects.getAttribute('ruleNameInput', 'value'); + let ruleName; + // Rule name input is different in serverless + if (await testSubjects.exists('ruleNameInput')) { + await testSubjects.setValue('ruleNameInput', newAlert); + ruleName = await testSubjects.getAttribute('ruleNameInput', 'value'); + } else { + await testSubjects.setValue('ruleDetailsNameInput', newAlert); + ruleName = await testSubjects.getAttribute('ruleDetailsNameInput', 'value'); + } return ruleName === newAlert; }); @@ -681,7 +694,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await comboBox.set('ruleFormConsumerSelect', 'Stack Rules'); } - await testSubjects.click('saveRuleButton'); + // Save rule button is different in serverless + if (await testSubjects.exists('saveRuleButton')) { + await testSubjects.click('saveRuleButton'); + } else { + await testSubjects.click('rulePageFooterSaveButton'); + } await retry.waitFor('confirmation modal', async () => { return await testSubjects.exists('confirmModalConfirmButton'); From 61540ad403056b439ca7c8303d4d398930fa675a Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Wed, 4 Dec 2024 21:41:30 -0700 Subject: [PATCH 3/4] Skip actual failing test --- .../discover_ml_uptime/discover/search_source_alert.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index 62c6650e9db75..84e5cc56a6514 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -222,7 +222,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const openDiscoverAlertFlyout = async () => { await testSubjects.click('discoverAlertsButton'); - await testSubjects.click('discoverCreateAlertButton'); + // Different create rule buttons in serverless + if (await testSubjects.exists('discoverCreateAlertButton')) { + await testSubjects.click('discoverCreateAlertButton'); + } else { + await testSubjects.click('discoverAppMenuCustomThresholdRule'); + } }; const openManagementAlertFlyout = async () => { @@ -427,7 +432,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { outputDataViewId = outputDataViewResponse.body.data_view.id; }); - it('should show time field validation error', async () => { + // Failing: https://github.com/elastic/kibana/issues/203045 + it.skip('should show time field validation error', async () => { await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.waitUntilSearchingHasFinished(); await dataViews.switchToAndValidate(SOURCE_DATA_VIEW); From 3cfd4ffcbfd2c59044a71e3099ee8af6c664cb17 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Thu, 5 Dec 2024 14:29:12 -0700 Subject: [PATCH 4/4] skip alert tests on serverless observability --- .../discover_ml_uptime/discover/search_source_alert.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index 84e5cc56a6514..1007c89cb171a 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -377,6 +377,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }; describe('Search source Alert', function () { + // Failing: https://github.com/elastic/kibana/issues/203045 + this.tags(['skipSvlOblt']); + before(async () => { await security.testUser.setRoles(['discover_alert']); await PageObjects.svlCommonPage.loginAsAdmin(); @@ -432,8 +435,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { outputDataViewId = outputDataViewResponse.body.data_view.id; }); - // Failing: https://github.com/elastic/kibana/issues/203045 - it.skip('should show time field validation error', async () => { + it('should show time field validation error', async () => { await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.waitUntilSearchingHasFinished(); await dataViews.switchToAndValidate(SOURCE_DATA_VIEW);