[8.15] [Detection Engine] Addresses Flakiness in ML FTR tests (#188155)…

… (#188259) # Backport This will backport the following commits from `main` to `8.15`: - [[Detection Engine] Addresses Flakiness in ML FTR tests (#188155)](#188155)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)
elastic · Jul 12, 2024 · 3867757 · 3867757
1 parent bc5b5da
commit 3867757
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 9 deletions.
diff --git a/...gine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts b/...gine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts
@@ -40,12 +40,12 @@ import {
   importFile,
 } from '../../../../../lists_and_exception_lists/utils';
 import {
-  executeSetupModuleRequest,
   forceStartDatafeeds,
   getAlerts,
   getPreviewAlerts,
   previewRule,
   previewRuleWithExceptionEntries,
+  setupMlModulesWithRetry,
 } from '../../../../utils';
 import {
   createRule,
@@ -86,13 +86,12 @@ export default ({ getService }: FtrProviderContext) => {
     rule_id: 'ml-rule-id',
   };
 
-  // FLAKY: https://github.com/elastic/kibana/issues/171426
-  describe.skip('@ess @serverless @serverlessQA Machine learning type rules', () => {
+  describe('@ess @serverless @serverlessQA Machine learning type rules', () => {
     before(async () => {
       // Order is critical here: auditbeat data must be loaded before attempting to start the ML job,
       // as the job looks for certain indices on start
       await esArchiver.load(auditPath);
-      await executeSetupModuleRequest({ module: siemModule, rspCode: 200, supertest });
+      await setupMlModulesWithRetry({ module: siemModule, supertest, retry });
       await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest });
       await esArchiver.load('x-pack/test/functional/es_archives/security_solution/anomalies');
     });

diff --git a/...n_logic/trial_license_complete_tier/execution_logic/machine_learning_alert_suppression.ts b/...n_logic/trial_license_complete_tier/execution_logic/machine_learning_alert_suppression.ts
@@ -27,7 +27,6 @@ import { EsArchivePathBuilder } from '../../../../../../es_archive_path_builder'
 import { FtrProviderContext } from '../../../../../../ftr_provider_context';
 import {
   dataGeneratorFactory,
-  executeSetupModuleRequest,
   forceStartDatafeeds,
   getAlerts,
   getOpenAlerts,
@@ -36,6 +35,7 @@ import {
   previewRule,
   previewRuleWithExceptionEntries,
   setAlertStatus,
+  setupMlModulesWithRetry,
 } from '../../../../utils';
 import {
   createRule,
@@ -51,6 +51,7 @@ export default ({ getService }: FtrProviderContext) => {
   const es = getService('es');
   const log = getService('log');
   const config = getService('config');
+  const retry = getService('retry');
 
   const isServerless = config.get('serverless');
   const dataPathBuilder = new EsArchivePathBuilder(isServerless);
@@ -87,14 +88,13 @@ export default ({ getService }: FtrProviderContext) => {
   // The tests described in this file rely on the
   // 'alertSuppressionForMachineLearningRuleEnabled' feature flag, and are thus
   // skipped in MKI
-  // Failing: See https://github.com/elastic/kibana/issues/187478
-  describe.skip('@ess @serverless @skipInServerlessMKI Machine Learning Detection Rule - Alert Suppression', () => {
+  describe('@ess @serverless @skipInServerlessMKI Machine Learning Detection Rule - Alert Suppression', () => {
     describe('with an active ML Job', () => {
       before(async () => {
         // Order is critical here: auditbeat data must be loaded before attempting to start the ML job,
         // as the job looks for certain indices on start
         await esArchiver.load(auditbeatArchivePath);
-        await executeSetupModuleRequest({ module: mlModuleName, rspCode: 200, supertest });
+        await setupMlModulesWithRetry({ module: mlModuleName, retry, supertest });
         await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest });
         await esArchiver.load('x-pack/test/functional/es_archives/security_solution/anomalies');
         await deleteAllAnomalies(log, es);

diff --git a/...egration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts b/...egration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts
@@ -6,9 +6,18 @@
  */
 
 import type SuperTest from 'supertest';
+import { RetryService } from '@kbn/ftr-common-functional-services';
 import { ML_GROUP_ID } from '@kbn/security-solution-plugin/common/constants';
 import { getCommonRequestHeader } from '../../../../../functional/services/ml/common_api';
 
+interface ModuleJob {
+  id: string;
+  success: boolean;
+  error?: {
+    status: number;
+  };
+}
+
 export const executeSetupModuleRequest = async ({
   module,
   rspCode,
@@ -17,7 +26,7 @@ export const executeSetupModuleRequest = async ({
   module: string;
   rspCode: number;
   supertest: SuperTest.Agent;
-}) => {
+}): Promise<{ jobs: ModuleJob[] }> => {
   const { body } = await supertest
     .post(`/internal/ml/modules/setup/${module}`)
     .set(getCommonRequestHeader('1'))
@@ -34,6 +43,35 @@ export const executeSetupModuleRequest = async ({
   return body;
 };
 
+export const setupMlModulesWithRetry = async ({
+  module,
+  retry,
+  supertest,
+}: {
+  module: string;
+  retry: RetryService;
+  supertest: SuperTest.Agent;
+}) =>
+  retry.try(async () => {
+    const response = await executeSetupModuleRequest({
+      module,
+      rspCode: 200,
+      supertest,
+    });
+
+    const allJobsSucceeded = response?.jobs.every((job) => {
+      return job.success || (job.error?.status && job.error.status < 500);
+    });
+
+    if (!allJobsSucceeded) {
+      throw new Error(
+        `Expected all jobs to set up successfully, but got ${JSON.stringify(response)}`
+      );
+    }
+
+    return response;
+  });
+
 export const forceStartDatafeeds = async ({
   jobId,
   rspCode,