From 19dde645c8c352d11efdfe8dc13e166103dcdb35 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 23 Oct 2024 12:51:08 +0100 Subject: [PATCH] [8.x] [Entity Store] [FTR Tests] Fix flakiness + poll for engine started on setup (#196564) (#197050) # Backport This will backport the following commits from `main` to `8.x`: - [[Entity Store] [FTR Tests] Fix flakiness + poll for engine started on setup (#196564)](https://github.com/elastic/kibana/pull/196564) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --- .../elasticsearch_assets/enrich_policy.ts | 30 ++++- .../entity_store/entity_store_data_client.ts | 5 +- .../trial_license_complete_tier/engine.ts | 21 ++- .../engine_nondefault_spaces.ts | 18 +-- .../utils/elastic_asset_checker.ts | 127 +++++++++++------- .../entity_analytics/utils/entity_store.ts | 33 ++++- 6 files changed, 151 insertions(+), 83 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts index 7d6fc6fd8bc24..4b8ce594a6cb7 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts @@ -72,10 +72,36 @@ export const executeFieldRetentionEnrichPolicy = async ({ export const deleteFieldRetentionEnrichPolicy = async ({ unitedDefinition, esClient, + logger, + attempts = 5, + delayMs = 2000, }: { - esClient: ElasticsearchClient; unitedDefinition: DefinitionMetadata; + esClient: ElasticsearchClient; + logger: Logger; + attempts?: number; + delayMs?: number; }) => { const name = getFieldRetentionEnrichPolicyName(unitedDefinition); - return esClient.enrich.deletePolicy({ name }, { ignore: [404] }); + let currentAttempt = 1; + while (currentAttempt <= attempts) { + try { + await esClient.enrich.deletePolicy({ name }, { ignore: [404] }); + return; + } catch (e) { + // a 429 status code indicates that the enrich policy is being executed + if (currentAttempt === attempts || e.statusCode !== 429) { + logger.error( + `Error deleting enrich policy ${name}: ${e.message} after ${currentAttempt} attempts` + ); + throw e; + } + + logger.info( + `Enrich policy ${name} is being executed, waiting for it to finish before deleting` + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + currentAttempt++; + } + } }; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index ce695f417b045..2cb119e6d37fe 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -247,7 +247,7 @@ export class EntityStoreDataClient { logger, taskManager, }); - logger.info(`Entity store initialized`); + logger.info(`Entity store initialized for ${entityType}`); return updated; } catch (err) { @@ -364,6 +364,7 @@ export class EntityStoreDataClient { await deleteFieldRetentionEnrichPolicy({ unitedDefinition, esClient: this.esClient, + logger, }); if (deleteData) { @@ -464,7 +465,7 @@ export class EntityStoreDataClient { originalStatus === ENGINE_STATUS.UPDATING ) { throw new Error( - `Error updating entity store: There is an changes already in progress for engine ${id}` + `Error updating entity store: There are changes already in progress for engine ${id}` ); } diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index 63a1530744440..f51fbd15ceead 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -14,9 +14,7 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const utils = EntityStoreUtils(getService); - - // Failing: See https://github.com/elastic/kibana/issues/196526 - describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => { + describe('@ess @skipInServerlessMKI Entity Store Engine APIs', () => { const dataView = dataViewRouteHelpersFactory(supertest); before(async () => { @@ -34,22 +32,19 @@ export default ({ getService }: FtrProviderContext) => { }); it('should have installed the expected user resources', async () => { - await utils.initEntityEngineForEntityType('user'); + await utils.initEntityEngineForEntityTypesAndWait(['user']); await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); await utils.expectEngineAssetsExist('host'); }); }); describe('get and list', () => { before(async () => { - await Promise.all([ - utils.initEntityEngineForEntityType('host'), - utils.initEntityEngineForEntityType('user'), - ]); + await utils.initEntityEngineForEntityTypesAndWait(['host', 'user']); }); after(async () => { @@ -119,7 +114,7 @@ export default ({ getService }: FtrProviderContext) => { describe('start and stop', () => { before(async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); }); after(async () => { @@ -161,7 +156,7 @@ export default ({ getService }: FtrProviderContext) => { describe('delete', () => { it('should delete the host entity engine', async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); await api .deleteEntityEngine({ @@ -174,7 +169,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should delete the user entity engine', async () => { - await utils.initEntityEngineForEntityType('user'); + await utils.initEntityEngineForEntityTypesAndWait(['user']); await api .deleteEntityEngine({ @@ -189,7 +184,7 @@ export default ({ getService }: FtrProviderContext) => { describe('apply_dataview_indices', () => { before(async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); }); after(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index 481f7aa4056f6..64809533fec7b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -18,8 +18,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { const supertest = getService('supertest'); const utils = EntityStoreUtils(getService, namespace); - // Failing: See https://github.com/elastic/kibana/issues/196546 - describe.skip('@ess Entity Store Engine APIs in non-default space', () => { + describe('@ess Entity Store Engine APIs in non-default space', () => { const dataView = dataViewRouteHelpersFactory(supertest, namespace); before(async () => { @@ -43,22 +42,19 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { }); it('should have installed the expected user resources', async () => { - await utils.initEntityEngineForEntityType('user'); + await utils.initEntityEngineForEntityTypesAndWait(['user']); await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); await utils.expectEngineAssetsExist('host'); }); }); describe('get and list', () => { before(async () => { - await Promise.all([ - utils.initEntityEngineForEntityType('host'), - utils.initEntityEngineForEntityType('user'), - ]); + await utils.initEntityEngineForEntityTypesAndWait(['host', 'user']); }); after(async () => { @@ -134,7 +130,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { describe('start and stop', () => { before(async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); }); after(async () => { @@ -188,7 +184,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { describe('delete', () => { it('should delete the host entity engine', async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); await api .deleteEntityEngine( @@ -204,7 +200,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { }); it('should delete the user entity engine', async () => { - await utils.initEntityEngineForEntityType('user'); + await utils.initEntityEngineForEntityTypesAndWait(['user']); await api .deleteEntityEngine( diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts index 8e8635cefc26b..c0dddb4ddb093 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts @@ -9,6 +9,8 @@ import { FtrProviderContext } from '@kbn/ftr-common-functional-services'; export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getService']) => { const es = getService('es'); + const retry = getService('retry'); + const log = getService('log'); const expectTransformExists = async (transformId: string) => { return expectTransformStatus(transformId, true); @@ -18,45 +20,43 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe return expectTransformStatus(transformId, false); }; - const expectTransformStatus = async ( - transformId: string, - exists: boolean, - attempts: number = 5, - delayMs: number = 2000 - ) => { - let currentAttempt = 1; - while (currentAttempt <= attempts) { - try { - await es.transform.getTransform({ transform_id: transformId }); - if (!exists) { - throw new Error(`Expected transform ${transformId} to not exist, but it does`); + const expectTransformStatus = async (transformId: string, exists: boolean) => { + await retry.waitForWithTimeout( + `transform ${transformId} to ${exists ? 'exist' : 'not exist'}`, + 10_000, + async () => { + try { + await es.transform.getTransform({ transform_id: transformId }); + return exists; + } catch (e) { + log.debug(`Transform ${transformId} not found: ${e}`); + return !exists; } - return; // Transform exists, exit the loop - } catch (e) { - if (currentAttempt === attempts) { - if (exists) { - throw new Error(`Expected transform ${transformId} to exist, but it does not: ${e}`); - } else { - return; // Transform does not exist, exit the loop - } - } - await new Promise((resolve) => setTimeout(resolve, delayMs)); - currentAttempt++; } - } + ); }; const expectEnrichPolicyStatus = async (policyId: string, exists: boolean) => { - try { - await es.enrich.getPolicy({ name: policyId }); - if (!exists) { - throw new Error(`Expected enrich policy ${policyId} to not exist, but it does`); - } - } catch (e) { - if (exists) { - throw new Error(`Expected enrich policy ${policyId} to exist, but it does not: ${e}`); + await retry.waitForWithTimeout( + `enrich policy ${policyId} to ${exists ? 'exist' : 'not exist'}`, + 20_000, + async () => { + try { + const res = await es.enrich.getPolicy({ name: policyId }); + const policy = res.policies?.[0]; + if (policy) { + log.debug(`Enrich policy ${policyId} found: ${JSON.stringify(res)}`); + return exists; + } else { + log.debug(`Enrich policy ${policyId} not found: ${JSON.stringify(res)}`); + return !exists; + } + } catch (e) { + log.debug(`Enrich policy ${policyId} not found: ${e}`); + return !exists; + } } - } + ); }; const expectEnrichPolicyExists = async (policyId: string) => @@ -66,18 +66,19 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe expectEnrichPolicyStatus(policyId, false); const expectComponentTemplatStatus = async (templateName: string, exists: boolean) => { - try { - await es.cluster.getComponentTemplate({ name: templateName }); - if (!exists) { - throw new Error(`Expected component template ${templateName} to not exist, but it does`); - } - } catch (e) { - if (exists) { - throw new Error( - `Expected component template ${templateName} to exist, but it does not: ${e}` - ); + await retry.waitForWithTimeout( + `component template ${templateName} to ${exists ? 'exist' : 'not exist'}`, + 10_000, + async () => { + try { + await es.cluster.getComponentTemplate({ name: templateName }); + return exists; // Component template exists + } catch (e) { + log.debug(`Component template ${templateName} not found: ${e}`); + return !exists; // Component template does not exist + } } - } + ); }; const expectComponentTemplateExists = async (templateName: string) => @@ -87,23 +88,45 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe expectComponentTemplatStatus(templateName, false); const expectIngestPipelineStatus = async (pipelineId: string, exists: boolean) => { + await retry.waitForWithTimeout( + `ingest pipeline ${pipelineId} to ${exists ? 'exist' : 'not exist'}`, + 10_000, + async () => { + try { + await es.ingest.getPipeline({ id: pipelineId }); + return exists; // Ingest pipeline exists + } catch (e) { + log.debug(`Ingest pipeline ${pipelineId} not found: ${e}`); + return !exists; // Ingest pipeline does not exist + } + } + ); + }; + + const expectIngestPipelineExists = async (pipelineId: string) => + expectIngestPipelineStatus(pipelineId, true); + + const expectIngestPipelineNotFound = async (pipelineId: string) => + expectIngestPipelineStatus(pipelineId, false); + + const expectIndexStatus = async (indexName: string, exists: boolean) => { try { - await es.ingest.getPipeline({ id: pipelineId }); + await es.indices.get({ index: indexName }); if (!exists) { - throw new Error(`Expected ingest pipeline ${pipelineId} to not exist, but it does`); + throw new Error(`Expected index ${indexName} to not exist, but it does`); } } catch (e) { if (exists) { - throw new Error(`Expected ingest pipeline ${pipelineId} to exist, but it does not: ${e}`); + throw new Error(`Expected index ${indexName} to exist, but it does not: ${e}`); } } }; - const expectIngestPipelineExists = async (pipelineId: string) => - expectIngestPipelineStatus(pipelineId, true); + const expectEntitiesIndexExists = async (entityType: string, namespace: string) => + expectIndexStatus(`.entities.v1.latest.security_${entityType}_${namespace}`, true); - const expectIngestPipelineNotFound = async (pipelineId: string) => - expectIngestPipelineStatus(pipelineId, false); + const expectEntitiesIndexNotFound = async (entityType: string, namespace: string) => + expectIndexStatus(`.entities.v1.latest.security_${entityType}_${namespace}`, false); return { expectComponentTemplateExists, @@ -112,6 +135,8 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe expectEnrichPolicyNotFound, expectIngestPipelineExists, expectIngestPipelineNotFound, + expectEntitiesIndexExists, + expectEntitiesIndexNotFound, expectTransformExists, expectTransformNotFound, }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 24c1434b5e4a5..029103425af68 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -17,6 +17,7 @@ export const EntityStoreUtils = ( const api = getService('securitySolutionApi'); const es = getService('es'); const log = getService('log'); + const retry = getService('retry'); const { expectTransformExists, expectTransformNotFound, @@ -26,6 +27,8 @@ export const EntityStoreUtils = ( expectComponentTemplateNotFound, expectIngestPipelineExists, expectIngestPipelineNotFound, + expectEntitiesIndexExists, + expectEntitiesIndexNotFound, } = elasticAssetCheckerFactory(getService); log.debug(`EntityStoreUtils namespace: ${namespace}`); @@ -48,7 +51,7 @@ export const EntityStoreUtils = ( } }; - const initEntityEngineForEntityType = async (entityType: EntityType) => { + const _initEntityEngineForEntityType = async (entityType: EntityType) => { log.info( `Initializing engine for entity type ${entityType} in namespace ${namespace || 'default'}` ); @@ -68,6 +71,22 @@ export const EntityStoreUtils = ( expect(res.status).to.eql(200); }; + const initEntityEngineForEntityTypesAndWait = async (entityTypes: EntityType[]) => { + await Promise.all(entityTypes.map((entityType) => _initEntityEngineForEntityType(entityType))); + + await retry.waitForWithTimeout( + `Engines to start for entity types: ${entityTypes.join(', ')}`, + 60_000, + async () => { + const { body } = await api.listEntityEngines(namespace).expect(200); + if (body.engines.every((engine: any) => engine.status === 'started')) { + return true; + } + return false; + } + ); + }; + const expectTransformStatus = async ( transformId: string, exists: boolean, @@ -98,21 +117,27 @@ export const EntityStoreUtils = ( const expectEngineAssetsExist = async (entityType: EntityType) => { await expectTransformExists(`entities-v1-latest-security_${entityType}_${namespace}`); - await expectEnrichPolicyExists(`entity_store_field_retention_${entityType}_${namespace}_v1`); + await expectEnrichPolicyExists( + `entity_store_field_retention_${entityType}_${namespace}_v1.0.0` + ); await expectComponentTemplateExists(`security_${entityType}_${namespace}-latest@platform`); await expectIngestPipelineExists(`security_${entityType}_${namespace}-latest@platform`); + await expectEntitiesIndexExists(entityType, namespace); }; const expectEngineAssetsDoNotExist = async (entityType: EntityType) => { await expectTransformNotFound(`entities-v1-latest-security_${entityType}_${namespace}`); - await expectEnrichPolicyNotFound(`entity_store_field_retention_${entityType}_${namespace}_v1`); + await expectEnrichPolicyNotFound( + `entity_store_field_retention_${entityType}_${namespace}_v1.0.0` + ); await expectComponentTemplateNotFound(`security_${entityType}_${namespace}-latest@platform`); await expectIngestPipelineNotFound(`security_${entityType}_${namespace}-latest@platform`); + await expectEntitiesIndexNotFound(entityType, namespace); }; return { cleanEngines, - initEntityEngineForEntityType, + initEntityEngineForEntityTypesAndWait, expectTransformStatus, expectEngineAssetsExist, expectEngineAssetsDoNotExist,