diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 90dd34d86af21..8e5c2b8f1b127 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -91,7 +91,7 @@ describe('checking migration metadata changes on all registered SO types', () => "endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", - "entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9", + "entity-definition": "e3811fd5fbb878d170067c0d6897a2e63010af36", "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88", "entity-engine-status": "0738aa1a06d3361911740f8f166071ea43a00927", "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd", diff --git a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap index 9210d3b9991cf..766ce1c70ac3a 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap +++ b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap @@ -78,7 +78,8 @@ exports[`schemas metadataSchema should parse successfully with a source and desi Object { "data": Object { "aggregation": Object { - "limit": 1000, + "limit": 10, + "lookbackPeriod": undefined, "type": "terms", }, "destination": "hostName", @@ -92,7 +93,8 @@ exports[`schemas metadataSchema should parse successfully with an valid string 1 Object { "data": Object { "aggregation": Object { - "limit": 1000, + "limit": 10, + "lookbackPeriod": undefined, "type": "terms", }, "destination": "host.name", @@ -106,7 +108,8 @@ exports[`schemas metadataSchema should parse successfully with just a source 1`] Object { "data": Object { "aggregation": Object { - "limit": 1000, + "limit": 10, + "lookbackPeriod": undefined, "type": "terms", }, "destination": "host.name", @@ -120,7 +123,8 @@ exports[`schemas metadataSchema should parse successfully with valid object 1`] Object { "data": Object { "aggregation": Object { - "limit": 1000, + "limit": 10, + "lookbackPeriod": undefined, "type": "terms", }, "destination": "hostName", diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts index 1a737ac3f4d9b..210e34943bd40 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { durationSchema, metadataSchema, semVerSchema, historySettingsSchema } from './common'; +import { durationSchema, metadataSchema, semVerSchema } from './common'; describe('schemas', () => { describe('metadataSchema', () => { @@ -66,7 +66,7 @@ describe('schemas', () => { expect(result.data).toEqual({ source: 'host.name', destination: 'hostName', - aggregation: { type: 'terms', limit: 1000 }, + aggregation: { type: 'terms', limit: 10, lookbackPeriod: undefined }, }); }); @@ -139,30 +139,4 @@ describe('schemas', () => { expect(result).toMatchSnapshot(); }); }); - - describe('historySettingsSchema', () => { - it('should return default values when not defined', () => { - let result = historySettingsSchema.safeParse(undefined); - expect(result.success).toBeTruthy(); - expect(result.data).toEqual({ lookbackPeriod: '1h' }); - - result = historySettingsSchema.safeParse({ syncDelay: '1m' }); - expect(result.success).toBeTruthy(); - expect(result.data).toEqual({ syncDelay: '1m', lookbackPeriod: '1h' }); - }); - - it('should return user defined values when defined', () => { - const result = historySettingsSchema.safeParse({ - lookbackPeriod: '30m', - syncField: 'event.ingested', - syncDelay: '5m', - }); - expect(result.success).toBeTruthy(); - expect(result.data).toEqual({ - lookbackPeriod: '30m', - syncField: 'event.ingested', - syncDelay: '5m', - }); - }); - }); }); diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index aa54dbd16c9aa..caecf48d88aac 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -85,7 +85,11 @@ export const keyMetricSchema = z.object({ export type KeyMetric = z.infer; export const metadataAggregation = z.union([ - z.object({ type: z.literal('terms'), limit: z.number().default(1000) }), + z.object({ + type: z.literal('terms'), + limit: z.number().default(10), + lookbackPeriod: z.optional(durationSchema), + }), z.object({ type: z.literal('top_value'), sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])), @@ -99,13 +103,13 @@ export const metadataSchema = z destination: z.optional(z.string()), aggregation: z .optional(metadataAggregation) - .default({ type: z.literal('terms').value, limit: 1000 }), + .default({ type: z.literal('terms').value, limit: 10, lookbackPeriod: undefined }), }) .or( z.string().transform((value) => ({ source: value, destination: value, - aggregation: { type: z.literal('terms').value, limit: 1000 }, + aggregation: { type: z.literal('terms').value, limit: 10, lookbackPeriod: undefined }, })) ) .transform((metadata) => ({ diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts index eae6873356c14..3eb87a797ef21 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts @@ -35,7 +35,6 @@ export const entityLatestSchema = z entity: entityBaseSchema.merge( z.object({ lastSeenTimestamp: z.string(), - firstSeenTimestamp: z.string(), }) ), }) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts index 74be36cc5d802..d9d8e6b610013 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts @@ -14,8 +14,6 @@ import { durationSchema, identityFieldsSchema, semVerSchema, - historySettingsSchema, - durationSchemaWithMinimum, } from './common'; export const entityDefinitionSchema = z.object({ @@ -32,22 +30,17 @@ export const entityDefinitionSchema = z.object({ metrics: z.optional(z.array(keyMetricSchema)), staticFields: z.optional(z.record(z.string(), z.string())), managed: z.optional(z.boolean()).default(false), - history: z.object({ + latest: z.object({ timestampField: z.string(), - interval: durationSchemaWithMinimum(1), - settings: historySettingsSchema, + lookbackPeriod: z.optional(durationSchema).default('24h'), + settings: z.optional( + z.object({ + syncField: z.optional(z.string()), + syncDelay: z.optional(durationSchema), + frequency: z.optional(durationSchema), + }) + ), }), - latest: z.optional( - z.object({ - settings: z.optional( - z.object({ - syncField: z.optional(z.string()), - syncDelay: z.optional(durationSchema), - frequency: z.optional(durationSchema), - }) - ), - }) - ), installStatus: z.optional( z.union([ z.literal('installing'), @@ -57,6 +50,18 @@ export const entityDefinitionSchema = z.object({ ]) ), installStartedAt: z.optional(z.string()), + installedComponents: z.optional( + z.array( + z.object({ + type: z.union([ + z.literal('transform'), + z.literal('ingest_pipeline'), + z.literal('template'), + ]), + id: z.string(), + }) + ) + ), }); export const entityDefinitionUpdateSchema = entityDefinitionSchema @@ -69,7 +74,7 @@ export const entityDefinitionUpdateSchema = entityDefinitionSchema .partial() .merge( z.object({ - history: z.optional(entityDefinitionSchema.shape.history.partial()), + latest: z.optional(entityDefinitionSchema.shape.latest.partial()), version: semVerSchema, }) ); diff --git a/x-pack/plugins/entity_manager/common/constants_entities.ts b/x-pack/plugins/entity_manager/common/constants_entities.ts index c53847afbb548..c17e6f33918c6 100644 --- a/x-pack/plugins/entity_manager/common/constants_entities.ts +++ b/x-pack/plugins/entity_manager/common/constants_entities.ts @@ -33,8 +33,6 @@ export const ENTITY_LATEST_PREFIX_V1 = `${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_LATEST}` as const; // Transform constants -export const ENTITY_DEFAULT_HISTORY_FREQUENCY = '1m'; -export const ENTITY_DEFAULT_HISTORY_SYNC_DELAY = '60s'; -export const ENTITY_DEFAULT_LATEST_FREQUENCY = '30s'; -export const ENTITY_DEFAULT_LATEST_SYNC_DELAY = '1s'; -export const ENTITY_DEFAULT_METADATA_LIMIT = 1000; +export const ENTITY_DEFAULT_LATEST_FREQUENCY = '1m'; +export const ENTITY_DEFAULT_LATEST_SYNC_DELAY = '60s'; +export const ENTITY_DEFAULT_METADATA_LIMIT = 10; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts index 6ce76e127c8e8..e3356c4826ae8 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts @@ -20,9 +20,9 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition = indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'], identityFields: ['container.id'], displayNameTemplate: '{{container.id}}', - history: { + latest: { timestampField: '@timestamp', - interval: '5m', + lookbackPeriod: '10m', settings: { frequency: '5m', }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts index 56f83d5fbaed6..5d7a30093419e 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts @@ -19,9 +19,9 @@ export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefin indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'], identityFields: ['host.name'], displayNameTemplate: '{{host.name}}', - history: { + latest: { timestampField: '@timestamp', - interval: '5m', + lookbackPeriod: '10m', settings: { frequency: '5m', }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts index 6caa209da02ca..d6aa4d08ad221 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts @@ -18,11 +18,10 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition = type: 'service', managed: true, indexPatterns: ['logs-*', 'filebeat*', 'traces-apm*'], - history: { + latest: { timestampField: '@timestamp', - interval: '1m', + lookbackPeriod: '10m', settings: { - lookbackPeriod: '10m', frequency: '2m', syncDelay: '2m', }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts index 360f416cd5a00..0b3900363c0c8 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts @@ -7,46 +7,15 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { - generateHistoryIngestPipelineId, - generateLatestIngestPipelineId, -} from './helpers/generate_component_id'; +import { generateLatestIngestPipelineId } from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; -import { generateHistoryProcessors } from './ingest_pipeline/generate_history_processors'; import { generateLatestProcessors } from './ingest_pipeline/generate_latest_processors'; -export async function createAndInstallHistoryIngestPipeline( +export async function createAndInstallIngestPipelines( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger -) { - try { - const historyProcessors = generateHistoryProcessors(definition); - const historyId = generateHistoryIngestPipelineId(definition); - await retryTransientEsErrors( - () => - esClient.ingest.putPipeline({ - id: historyId, - processors: historyProcessors, - _meta: { - definitionVersion: definition.version, - managed: definition.managed, - }, - }), - { logger } - ); - } catch (e) { - logger.error( - `Cannot create entity history ingest pipelines for [${definition.id}] entity defintion` - ); - throw e; - } -} -export async function createAndInstallLatestIngestPipeline( - esClient: ElasticsearchClient, - definition: EntityDefinition, - logger: Logger -) { +): Promise> { try { const latestProcessors = generateLatestProcessors(definition); const latestId = generateLatestIngestPipelineId(definition); @@ -62,9 +31,10 @@ export async function createAndInstallLatestIngestPipeline( }), { logger } ); + return [{ type: 'ingest_pipeline', id: latestId }]; } catch (e) { logger.error( - `Cannot create entity latest ingest pipelines for [${definition.id}] entity defintion` + `Cannot create entity latest ingest pipelines for [${definition.id}] entity definition` ); throw e; } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts index d6379773479fc..779e0994a33b8 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts @@ -9,57 +9,20 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; import { retryTransientEsErrors } from './helpers/retry'; import { generateLatestTransform } from './transform/generate_latest_transform'; -import { - generateBackfillHistoryTransform, - generateHistoryTransform, -} from './transform/generate_history_transform'; -export async function createAndInstallHistoryTransform( +export async function createAndInstallTransforms( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger -) { - try { - const historyTransform = generateHistoryTransform(definition); - await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), { - logger, - }); - } catch (e) { - logger.error(`Cannot create entity history transform for [${definition.id}] entity definition`); - throw e; - } -} - -export async function createAndInstallHistoryBackfillTransform( - esClient: ElasticsearchClient, - definition: EntityDefinition, - logger: Logger -) { - try { - const historyTransform = generateBackfillHistoryTransform(definition); - await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), { - logger, - }); - } catch (e) { - logger.error( - `Cannot create entity history backfill transform for [${definition.id}] entity definition` - ); - throw e; - } -} - -export async function createAndInstallLatestTransform( - esClient: ElasticsearchClient, - definition: EntityDefinition, - logger: Logger -) { +): Promise> { try { const latestTransform = generateLatestTransform(definition); await retryTransientEsErrors(() => esClient.transform.putTransform(latestTransform), { logger, }); + return [{ type: 'transform', id: latestTransform.transform_id }]; } catch (e) { - logger.error(`Cannot create entity latest transform for [${definition.id}] entity definition`); + logger.error(`Cannot create entity history transform for [${definition.id}] entity definition`); throw e; } } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts b/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts index f4c46d8447d8f..a3b910dd4cb5e 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts @@ -7,24 +7,24 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { - generateHistoryIngestPipelineId, - generateLatestIngestPipelineId, -} from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; +import { generateLatestIngestPipelineId } from './helpers/generate_component_id'; -export async function deleteHistoryIngestPipeline( +export async function deleteIngestPipelines( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger ) { try { - const historyPipelineId = generateHistoryIngestPipelineId(definition); - await retryTransientEsErrors(() => - esClient.ingest.deletePipeline({ id: historyPipelineId }, { ignore: [404] }) + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'ingest_pipeline') + .map(({ id }) => + retryTransientEsErrors(() => esClient.ingest.deletePipeline({ id }, { ignore: [404] })) + ) ); } catch (e) { - logger.error(`Unable to delete history ingest pipeline [${definition.id}]: ${e}`); + logger.error(`Unable to delete ingest pipelines for definition [${definition.id}]: ${e}`); throw e; } } @@ -35,9 +35,11 @@ export async function deleteLatestIngestPipeline( logger: Logger ) { try { - const latestPipelineId = generateLatestIngestPipelineId(definition); await retryTransientEsErrors(() => - esClient.ingest.deletePipeline({ id: latestPipelineId }, { ignore: [404] }) + esClient.ingest.deletePipeline( + { id: generateLatestIngestPipelineId(definition) }, + { ignore: [404] } + ) ); } catch (e) { logger.error(`Unable to delete latest ingest pipeline [${definition.id}]: ${e}`); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts b/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts index a66c0998c014d..79b83998d38db 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts @@ -7,14 +7,8 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; - -import { - generateHistoryTransformId, - generateHistoryBackfillTransformId, - generateLatestTransformId, -} from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; +import { generateLatestTransformId } from './helpers/generate_component_id'; export async function deleteTransforms( esClient: ElasticsearchClient, @@ -22,37 +16,42 @@ export async function deleteTransforms( logger: Logger ) { try { - const historyTransformId = generateHistoryTransformId(definition); - const latestTransformId = generateLatestTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.deleteTransform( - { transform_id: historyTransformId, force: true }, - { ignore: [404] } - ), - { logger } + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'transform') + .map(({ id }) => + retryTransientEsErrors( + () => + esClient.transform.deleteTransform( + { transform_id: id, force: true }, + { ignore: [404] } + ), + { logger } + ) + ) ); - if (isBackfillEnabled(definition)) { - const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.deleteTransform( - { transform_id: historyBackfillTransformId, force: true }, - { ignore: [404] } - ), - { logger } - ); - } + } catch (e) { + logger.error(`Cannot delete transforms for definition [${definition.id}]: ${e}`); + throw e; + } +} + +export async function deleteLatestTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { await retryTransientEsErrors( () => esClient.transform.deleteTransform( - { transform_id: latestTransformId, force: true }, + { transform_id: generateLatestTransformId(definition), force: true }, { ignore: [404] } ), { logger } ); } catch (e) { - logger.error(`Cannot delete history transform [${definition.id}]: ${e}`); + logger.error(`Cannot delete latest transform for definition [${definition.id}]: ${e}`); throw e; } } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts index d1d84f27414af..cfbb5a5ef5556 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts @@ -10,18 +10,8 @@ import { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/serve import { EntityDefinition } from '@kbn/entities-schema'; import { NodesIngestTotal } from '@elastic/elasticsearch/lib/api/types'; import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; -import { - generateHistoryTransformId, - generateHistoryBackfillTransformId, - generateHistoryIngestPipelineId, - generateHistoryIndexTemplateId, - generateLatestTransformId, - generateLatestIngestPipelineId, - generateLatestIndexTemplateId, -} from './helpers/generate_component_id'; import { BUILT_IN_ID_PREFIX } from './built_in'; import { EntityDefinitionState, EntityDefinitionWithState } from './types'; -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; export async function findEntityDefinitions({ soClient, @@ -120,11 +110,9 @@ async function getTransformState({ definition: EntityDefinition; esClient: ElasticsearchClient; }) { - const transformIds = [ - generateHistoryTransformId(definition), - generateLatestTransformId(definition), - ...(isBackfillEnabled(definition) ? [generateHistoryBackfillTransformId(definition)] : []), - ]; + const transformIds = (definition.installedComponents ?? []) + .filter(({ type }) => type === 'transform') + .map(({ id }) => id); const transformStats = await Promise.all( transformIds.map((id) => esClient.transform.getTransformStats({ transform_id: id })) @@ -152,10 +140,10 @@ async function getIngestPipelineState({ definition: EntityDefinition; esClient: ElasticsearchClient; }) { - const ingestPipelineIds = [ - generateHistoryIngestPipelineId(definition), - generateLatestIngestPipelineId(definition), - ]; + const ingestPipelineIds = (definition.installedComponents ?? []) + .filter(({ type }) => type === 'ingest_pipeline') + .map(({ id }) => id); + const [ingestPipelines, ingestPipelinesStats] = await Promise.all([ esClient.ingest.getPipeline({ id: ingestPipelineIds.join(',') }, { ignore: [404] }), esClient.nodes.stats({ @@ -193,10 +181,9 @@ async function getIndexTemplatesState({ definition: EntityDefinition; esClient: ElasticsearchClient; }) { - const indexTemplatesIds = [ - generateLatestIndexTemplateId(definition), - generateHistoryIndexTemplateId(definition), - ]; + const indexTemplatesIds = (definition.installedComponents ?? []) + .filter(({ type }) => type === 'template') + .map(({ id }) => id); const templates = await Promise.all( indexTemplatesIds.map((id) => esClient.indices diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/calculate_offset.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/calculate_offset.ts deleted file mode 100644 index 3eba710561abf..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/calculate_offset.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; -import moment from 'moment'; -import { - ENTITY_DEFAULT_HISTORY_FREQUENCY, - ENTITY_DEFAULT_HISTORY_SYNC_DELAY, -} from '../../../../common/constants_entities'; - -const durationToSeconds = (dateMath: string) => { - const parts = dateMath.match(/(\d+)([m|s|h|d])/); - if (!parts) { - throw new Error(`Invalid date math supplied: ${dateMath}`); - } - const value = parseInt(parts[1], 10); - const unit = parts[2] as 'm' | 's' | 'h' | 'd'; - return moment.duration(value, unit).asSeconds(); -}; - -export function calculateOffset(definition: EntityDefinition) { - const syncDelay = durationToSeconds( - definition.history.settings.syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY - ); - const frequency = - durationToSeconds(definition.history.settings.frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY) * - 2; - - return syncDelay + frequency; -} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts index 5092e2caa5d78..b1e506150fb60 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts @@ -13,9 +13,8 @@ export const builtInEntityDefinition = entityDefinitionSchema.parse({ type: 'service', indexPatterns: ['kbn-data-forge-fake_stack.*'], managed: true, - history: { + latest: { timestampField: '@timestamp', - interval: '1m', }, identityFields: ['log.logger', { field: 'event.category', optional: true }], displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts index 940e209260c54..00ab9ac7759af 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts @@ -12,13 +12,12 @@ export const rawEntityDefinition = { name: 'Services for Admin Console', type: 'service', indexPatterns: ['kbn-data-forge-fake_stack.*'], - history: { + latest: { timestampField: '@timestamp', - interval: '1m', + lookbackPeriod: '10m', settings: { - lookbackPeriod: '10m', - frequency: '2m', - syncDelay: '2m', + frequency: '30s', + syncDelay: '10s', }, }, identityFields: ['log.logger', { field: 'event.category', optional: true }], diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts deleted file mode 100644 index 66a79825fbfb0..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entityDefinitionSchema } from '@kbn/entities-schema'; -export const entityDefinitionWithBackfill = entityDefinitionSchema.parse({ - id: 'admin-console-services-backfill', - version: '999.999.999', - name: 'Services for Admin Console', - type: 'service', - indexPatterns: ['kbn-data-forge-fake_stack.*'], - history: { - timestampField: '@timestamp', - interval: '1m', - settings: { - backfillSyncDelay: '15m', - backfillLookbackPeriod: '72h', - backfillFrequency: '5m', - }, - }, - identityFields: ['log.logger', { field: 'event.category', optional: true }], - displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}', - metadata: ['tags', 'host.name', 'host.os.name', { source: '_index', destination: 'sourceIndex' }], - metrics: [ - { - name: 'logRate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: 'log.level: *', - }, - ], - }, - { - name: 'errorRate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: 'log.level: "ERROR"', - }, - ], - }, - ], -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts index c24dcee1f8cf7..e841b1c8e23dd 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts @@ -6,5 +6,4 @@ */ export { entityDefinition } from './entity_definition'; -export { entityDefinitionWithBackfill } from './entity_definition_with_backfill'; export { builtInEntityDefinition } from './builtin_entity_definition'; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts deleted file mode 100644 index 4c34f5d3c0256..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; - -export function isBackfillEnabled(definition: EntityDefinition) { - return definition.history.settings.backfillSyncDelay != null; -} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap deleted file mode 100644 index c2e4605e5f909..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap +++ /dev/null @@ -1,327 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`generateHistoryProcessors(definition) should generate a valid pipeline for builtin definition 1`] = ` -Array [ - Object { - "set": Object { - "field": "event.ingested", - "value": "{{{_ingest.timestamp}}}", - }, - }, - Object { - "set": Object { - "field": "entity.type", - "value": "service", - }, - }, - Object { - "set": Object { - "field": "entity.definitionId", - "value": "builtin_mock_entity_definition", - }, - }, - Object { - "set": Object { - "field": "entity.definitionVersion", - "value": "1.0.0", - }, - }, - Object { - "set": Object { - "field": "entity.schemaVersion", - "value": "v1", - }, - }, - Object { - "set": Object { - "field": "entity.identityFields", - "value": Array [ - "log.logger", - "event.category", - ], - }, - }, - Object { - "script": Object { - "description": "Generated the entity.id field", - "source": "// This function will recursively collect all the values of a HashMap of HashMaps -Collection collectValues(HashMap subject) { - Collection values = new ArrayList(); - // Iterate through the values - for(Object value: subject.values()) { - // If the value is a HashMap, recurse - if (value instanceof HashMap) { - values.addAll(collectValues((HashMap) value)); - } else { - values.add(String.valueOf(value)); - } - } - return values; -} -// Create the string builder -StringBuilder entityId = new StringBuilder(); -if (ctx[\\"entity\\"][\\"identity\\"] != null) { - // Get the values as a collection - Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]); - // Convert to a list and sort - List sortedValues = new ArrayList(values); - Collections.sort(sortedValues); - // Create comma delimited string - for(String instanceValue: sortedValues) { - entityId.append(instanceValue); - entityId.append(\\":\\"); - } - // Assign the entity.id - ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\"; -}", - }, - }, - Object { - "fingerprint": Object { - "fields": Array [ - "entity.id", - ], - "method": "MurmurHash3", - "target_field": "entity.id", - }, - }, - Object { - "script": Object { - "source": "if (ctx.entity?.metadata?.tags != null) { - ctx.tags = ctx.entity.metadata.tags.keySet(); -} -if (ctx.entity?.metadata?.host?.name != null) { - if (ctx.host == null) { - ctx.host = new HashMap(); - } - ctx.host.name = ctx.entity.metadata.host.name.keySet(); -} -if (ctx.entity?.metadata?.host?.os?.name != null) { - if (ctx.host == null) { - ctx.host = new HashMap(); - } - if (ctx.host.os == null) { - ctx.host.os = new HashMap(); - } - ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet(); -} -if (ctx.entity?.metadata?.sourceIndex != null) { - ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet(); -}", - }, - }, - Object { - "remove": Object { - "field": "entity.metadata", - "ignore_missing": true, - }, - }, - Object { - "set": Object { - "field": "log.logger", - "if": "ctx.entity?.identity?.log?.logger != null", - "value": "{{entity.identity.log.logger}}", - }, - }, - Object { - "set": Object { - "field": "event.category", - "if": "ctx.entity?.identity?.event?.category != null", - "value": "{{entity.identity.event.category}}", - }, - }, - Object { - "remove": Object { - "field": "entity.identity", - "ignore_missing": true, - }, - }, - Object { - "date_index_name": Object { - "date_formats": Array [ - "UNIX_MS", - "ISO8601", - "yyyy-MM-dd'T'HH:mm:ss.SSSXX", - ], - "date_rounding": "M", - "field": "@timestamp", - "index_name_prefix": ".entities.v1.history.builtin_mock_entity_definition.", - }, - }, -] -`; - -exports[`generateHistoryProcessors(definition) should generate a valid pipeline for custom definition 1`] = ` -Array [ - Object { - "set": Object { - "field": "event.ingested", - "value": "{{{_ingest.timestamp}}}", - }, - }, - Object { - "set": Object { - "field": "entity.type", - "value": "service", - }, - }, - Object { - "set": Object { - "field": "entity.definitionId", - "value": "admin-console-services", - }, - }, - Object { - "set": Object { - "field": "entity.definitionVersion", - "value": "1.0.0", - }, - }, - Object { - "set": Object { - "field": "entity.schemaVersion", - "value": "v1", - }, - }, - Object { - "set": Object { - "field": "entity.identityFields", - "value": Array [ - "log.logger", - "event.category", - ], - }, - }, - Object { - "script": Object { - "description": "Generated the entity.id field", - "source": "// This function will recursively collect all the values of a HashMap of HashMaps -Collection collectValues(HashMap subject) { - Collection values = new ArrayList(); - // Iterate through the values - for(Object value: subject.values()) { - // If the value is a HashMap, recurse - if (value instanceof HashMap) { - values.addAll(collectValues((HashMap) value)); - } else { - values.add(String.valueOf(value)); - } - } - return values; -} -// Create the string builder -StringBuilder entityId = new StringBuilder(); -if (ctx[\\"entity\\"][\\"identity\\"] != null) { - // Get the values as a collection - Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]); - // Convert to a list and sort - List sortedValues = new ArrayList(values); - Collections.sort(sortedValues); - // Create comma delimited string - for(String instanceValue: sortedValues) { - entityId.append(instanceValue); - entityId.append(\\":\\"); - } - // Assign the entity.id - ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\"; -}", - }, - }, - Object { - "fingerprint": Object { - "fields": Array [ - "entity.id", - ], - "method": "MurmurHash3", - "target_field": "entity.id", - }, - }, - Object { - "script": Object { - "source": "if (ctx.entity?.metadata?.tags != null) { - ctx.tags = ctx.entity.metadata.tags.keySet(); -} -if (ctx.entity?.metadata?.host?.name != null) { - if (ctx.host == null) { - ctx.host = new HashMap(); - } - ctx.host.name = ctx.entity.metadata.host.name.keySet(); -} -if (ctx.entity?.metadata?.host?.os?.name != null) { - if (ctx.host == null) { - ctx.host = new HashMap(); - } - if (ctx.host.os == null) { - ctx.host.os = new HashMap(); - } - ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet(); -} -if (ctx.entity?.metadata?.sourceIndex != null) { - ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet(); -}", - }, - }, - Object { - "remove": Object { - "field": "entity.metadata", - "ignore_missing": true, - }, - }, - Object { - "set": Object { - "field": "log.logger", - "if": "ctx.entity?.identity?.log?.logger != null", - "value": "{{entity.identity.log.logger}}", - }, - }, - Object { - "set": Object { - "field": "event.category", - "if": "ctx.entity?.identity?.event?.category != null", - "value": "{{entity.identity.event.category}}", - }, - }, - Object { - "remove": Object { - "field": "entity.identity", - "ignore_missing": true, - }, - }, - Object { - "date_index_name": Object { - "date_formats": Array [ - "UNIX_MS", - "ISO8601", - "yyyy-MM-dd'T'HH:mm:ss.SSSXX", - ], - "date_rounding": "M", - "field": "@timestamp", - "index_name_prefix": ".entities.v1.history.admin-console-services.", - }, - }, - Object { - "pipeline": Object { - "ignore_missing_pipeline": true, - "name": "admin-console-services@platform", - }, - }, - Object { - "pipeline": Object { - "ignore_missing_pipeline": true, - "name": "admin-console-services-history@platform", - }, - }, - Object { - "pipeline": Object { - "ignore_missing_pipeline": true, - "name": "admin-console-services@custom", - }, - }, - Object { - "pipeline": Object { - "ignore_missing_pipeline": true, - "name": "admin-console-services-history@custom", - }, - }, -] -`; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap index f277b3ac84ab8..218deda422fe2 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap +++ b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap @@ -43,16 +43,60 @@ Array [ }, Object { "script": Object { - "source": "if (ctx.entity?.metadata?.tags.data != null) { + "description": "Generated the entity.id field", + "source": "// This function will recursively collect all the values of a HashMap of HashMaps +Collection collectValues(HashMap subject) { + Collection values = new ArrayList(); + // Iterate through the values + for(Object value: subject.values()) { + // If the value is a HashMap, recurse + if (value instanceof HashMap) { + values.addAll(collectValues((HashMap) value)); + } else { + values.add(String.valueOf(value)); + } + } + return values; +} +// Create the string builder +StringBuilder entityId = new StringBuilder(); +if (ctx[\\"entity\\"][\\"identity\\"] != null) { + // Get the values as a collection + Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]); + // Convert to a list and sort + List sortedValues = new ArrayList(values); + Collections.sort(sortedValues); + // Create comma delimited string + for(String instanceValue: sortedValues) { + entityId.append(instanceValue); + entityId.append(\\":\\"); + } + // Assign the entity.id + ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\"; +}", + }, + }, + Object { + "fingerprint": Object { + "fields": Array [ + "entity.id", + ], + "method": "MurmurHash3", + "target_field": "entity.id", + }, + }, + Object { + "script": Object { + "source": "if (ctx.entity?.metadata?.tags?.data != null) { ctx.tags = ctx.entity.metadata.tags.data.keySet(); } -if (ctx.entity?.metadata?.host?.name.data != null) { +if (ctx.entity?.metadata?.host?.name?.data != null) { if (ctx.host == null) { ctx.host = new HashMap(); } ctx.host.name = ctx.entity.metadata.host.name.data.keySet(); } -if (ctx.entity?.metadata?.host?.os?.name.data != null) { +if (ctx.entity?.metadata?.host?.os?.name?.data != null) { if (ctx.host == null) { ctx.host = new HashMap(); } @@ -61,7 +105,7 @@ if (ctx.entity?.metadata?.host?.os?.name.data != null) { } ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet(); } -if (ctx.entity?.metadata?.sourceIndex.data != null) { +if (ctx.entity?.metadata?.sourceIndex?.data != null) { ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet(); }", }, @@ -72,28 +116,18 @@ if (ctx.entity?.metadata?.sourceIndex.data != null) { "ignore_missing": true, }, }, - Object { - "dot_expander": Object { - "field": "log.logger", - "path": "entity.identity.log.logger.top_metric", - }, - }, Object { "set": Object { "field": "log.logger", - "value": "{{entity.identity.log.logger.top_metric.log.logger}}", - }, - }, - Object { - "dot_expander": Object { - "field": "event.category", - "path": "entity.identity.event.category.top_metric", + "if": "ctx.entity?.identity?.log?.logger != null", + "value": "{{entity.identity.log.logger}}", }, }, Object { "set": Object { "field": "event.category", - "value": "{{entity.identity.event.category.top_metric.event.category}}", + "if": "ctx.entity?.identity?.event?.category != null", + "value": "{{entity.identity.event.category}}", }, }, Object { @@ -160,16 +194,60 @@ Array [ }, Object { "script": Object { - "source": "if (ctx.entity?.metadata?.tags.data != null) { + "description": "Generated the entity.id field", + "source": "// This function will recursively collect all the values of a HashMap of HashMaps +Collection collectValues(HashMap subject) { + Collection values = new ArrayList(); + // Iterate through the values + for(Object value: subject.values()) { + // If the value is a HashMap, recurse + if (value instanceof HashMap) { + values.addAll(collectValues((HashMap) value)); + } else { + values.add(String.valueOf(value)); + } + } + return values; +} +// Create the string builder +StringBuilder entityId = new StringBuilder(); +if (ctx[\\"entity\\"][\\"identity\\"] != null) { + // Get the values as a collection + Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]); + // Convert to a list and sort + List sortedValues = new ArrayList(values); + Collections.sort(sortedValues); + // Create comma delimited string + for(String instanceValue: sortedValues) { + entityId.append(instanceValue); + entityId.append(\\":\\"); + } + // Assign the entity.id + ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\"; +}", + }, + }, + Object { + "fingerprint": Object { + "fields": Array [ + "entity.id", + ], + "method": "MurmurHash3", + "target_field": "entity.id", + }, + }, + Object { + "script": Object { + "source": "if (ctx.entity?.metadata?.tags?.data != null) { ctx.tags = ctx.entity.metadata.tags.data.keySet(); } -if (ctx.entity?.metadata?.host?.name.data != null) { +if (ctx.entity?.metadata?.host?.name?.data != null) { if (ctx.host == null) { ctx.host = new HashMap(); } ctx.host.name = ctx.entity.metadata.host.name.data.keySet(); } -if (ctx.entity?.metadata?.host?.os?.name.data != null) { +if (ctx.entity?.metadata?.host?.os?.name?.data != null) { if (ctx.host == null) { ctx.host = new HashMap(); } @@ -178,7 +256,7 @@ if (ctx.entity?.metadata?.host?.os?.name.data != null) { } ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet(); } -if (ctx.entity?.metadata?.sourceIndex.data != null) { +if (ctx.entity?.metadata?.sourceIndex?.data != null) { ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet(); }", }, @@ -189,28 +267,18 @@ if (ctx.entity?.metadata?.sourceIndex.data != null) { "ignore_missing": true, }, }, - Object { - "dot_expander": Object { - "field": "log.logger", - "path": "entity.identity.log.logger.top_metric", - }, - }, Object { "set": Object { "field": "log.logger", - "value": "{{entity.identity.log.logger.top_metric.log.logger}}", - }, - }, - Object { - "dot_expander": Object { - "field": "event.category", - "path": "entity.identity.event.category.top_metric", + "if": "ctx.entity?.identity?.log?.logger != null", + "value": "{{entity.identity.log.logger}}", }, }, Object { "set": Object { "field": "event.category", - "value": "{{entity.identity.event.category.top_metric.event.category}}", + "if": "ctx.entity?.identity?.event?.category != null", + "value": "{{entity.identity.event.category}}", }, }, Object { diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.test.ts deleted file mode 100644 index 717241b89143d..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entityDefinition, builtInEntityDefinition } from '../helpers/fixtures'; -import { generateHistoryProcessors } from './generate_history_processors'; - -describe('generateHistoryProcessors(definition)', () => { - it('should generate a valid pipeline for custom definition', () => { - const processors = generateHistoryProcessors(entityDefinition); - expect(processors).toMatchSnapshot(); - }); - - it('should generate a valid pipeline for builtin definition', () => { - const processors = generateHistoryProcessors(builtInEntityDefinition); - expect(processors).toMatchSnapshot(); - }); -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts deleted file mode 100644 index d51ab0be75db1..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1, MetadataField } from '@kbn/entities-schema'; -import { - initializePathScript, - cleanScript, -} from '../helpers/ingest_pipeline_script_processor_helpers'; -import { generateHistoryIndexName } from '../helpers/generate_component_id'; -import { isBuiltinDefinition } from '../helpers/is_builtin_definition'; - -function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { - if (aggregation.type === 'terms') { - return `ctx.entity.metadata.${destination}.keySet()`; - } else if (aggregation.type === 'top_value') { - return `ctx.entity.metadata.${destination}.top_value["${source}"]`; - } -} - -function mapDestinationToPainless(metadata: MetadataField) { - const field = metadata.destination; - return ` - ${initializePathScript(field)} - ctx.${field} = ${getMetadataSourceField(metadata)}; - `; -} - -function createMetadataPainlessScript(definition: EntityDefinition) { - if (!definition.metadata) { - return ''; - } - - return definition.metadata.reduce((acc, metadata) => { - const { destination, source } = metadata; - const optionalFieldPath = destination.replaceAll('.', '?.'); - - if (metadata.aggregation.type === 'terms') { - const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath} != null) { - ${mapDestinationToPainless(metadata)} - } - `; - return `${acc}\n${next}`; - } else if (metadata.aggregation.type === 'top_value') { - const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) { - ${mapDestinationToPainless(metadata)} - } - `; - return `${acc}\n${next}`; - } - - return acc; - }, ''); -} - -function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) { - return definition.identityFields.map((key) => ({ - set: { - if: `ctx.entity?.identity?.${key.field.replaceAll('.', '?.')} != null`, - field: key.field, - value: `{{entity.identity.${key.field}}}`, - }, - })); -} - -function getCustomIngestPipelines(definition: EntityDefinition) { - if (isBuiltinDefinition(definition)) { - return []; - } - - return [ - { - pipeline: { - ignore_missing_pipeline: true, - name: `${definition.id}@platform`, - }, - }, - { - pipeline: { - ignore_missing_pipeline: true, - name: `${definition.id}-history@platform`, - }, - }, - { - pipeline: { - ignore_missing_pipeline: true, - name: `${definition.id}@custom`, - }, - }, - { - pipeline: { - ignore_missing_pipeline: true, - name: `${definition.id}-history@custom`, - }, - }, - ]; -} - -export function generateHistoryProcessors(definition: EntityDefinition) { - return [ - { - set: { - field: 'event.ingested', - value: '{{{_ingest.timestamp}}}', - }, - }, - { - set: { - field: 'entity.type', - value: definition.type, - }, - }, - { - set: { - field: 'entity.definitionId', - value: definition.id, - }, - }, - { - set: { - field: 'entity.definitionVersion', - value: definition.version, - }, - }, - { - set: { - field: 'entity.schemaVersion', - value: ENTITY_SCHEMA_VERSION_V1, - }, - }, - { - set: { - field: 'entity.identityFields', - value: definition.identityFields.map((identityField) => identityField.field), - }, - }, - { - script: { - description: 'Generated the entity.id field', - source: cleanScript(` - // This function will recursively collect all the values of a HashMap of HashMaps - Collection collectValues(HashMap subject) { - Collection values = new ArrayList(); - // Iterate through the values - for(Object value: subject.values()) { - // If the value is a HashMap, recurse - if (value instanceof HashMap) { - values.addAll(collectValues((HashMap) value)); - } else { - values.add(String.valueOf(value)); - } - } - return values; - } - - // Create the string builder - StringBuilder entityId = new StringBuilder(); - - if (ctx["entity"]["identity"] != null) { - // Get the values as a collection - Collection values = collectValues(ctx["entity"]["identity"]); - - // Convert to a list and sort - List sortedValues = new ArrayList(values); - Collections.sort(sortedValues); - - // Create comma delimited string - for(String instanceValue: sortedValues) { - entityId.append(instanceValue); - entityId.append(":"); - } - - // Assign the entity.id - ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown"; - } - `), - }, - }, - { - fingerprint: { - fields: ['entity.id'], - target_field: 'entity.id', - method: 'MurmurHash3', - }, - }, - ...(definition.staticFields != null - ? Object.keys(definition.staticFields).map((field) => ({ - set: { field, value: definition.staticFields![field] }, - })) - : []), - ...(definition.metadata != null - ? [{ script: { source: cleanScript(createMetadataPainlessScript(definition)) } }] - : []), - { - remove: { - field: 'entity.metadata', - ignore_missing: true, - }, - }, - ...liftIdentityFieldsToDocumentRoot(definition), - { - remove: { - field: 'entity.identity', - ignore_missing: true, - }, - }, - { - date_index_name: { - field: '@timestamp', - index_name_prefix: `${generateHistoryIndexName(definition)}.`, - date_rounding: 'M', - date_formats: ['UNIX_MS', 'ISO8601', "yyyy-MM-dd'T'HH:mm:ss.SSSXX"], - }, - }, - ...getCustomIngestPipelines(definition), - ]; -} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index 16823221fffb3..0e3812de2e320 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -17,7 +17,7 @@ function getMetadataSourceField({ aggregation, destination, source }: MetadataFi if (aggregation.type === 'terms') { return `ctx.entity.metadata.${destination}.data.keySet()`; } else if (aggregation.type === 'top_value') { - return `ctx.entity.metadata.${destination}.top_value["${destination}"]`; + return `ctx.entity.metadata.${destination}.top_value["${source}"]`; } } @@ -35,19 +35,19 @@ function createMetadataPainlessScript(definition: EntityDefinition) { } return definition.metadata.reduce((acc, metadata) => { - const destination = metadata.destination; + const { destination, source } = metadata; const optionalFieldPath = destination.replaceAll('.', '?.'); if (metadata.aggregation.type === 'terms') { const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) { + if (ctx.entity?.metadata?.${optionalFieldPath}?.data != null) { ${mapDestinationToPainless(metadata)} } `; return `${acc}\n${next}`; } else if (metadata.aggregation.type === 'top_value') { const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${destination}"] != null) { + if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) { ${mapDestinationToPainless(metadata)} } `; @@ -59,30 +59,13 @@ function createMetadataPainlessScript(definition: EntityDefinition) { } function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) { - return definition.identityFields - .map((identityField) => { - const setProcessor = { - set: { - field: identityField.field, - value: `{{entity.identity.${identityField.field}.top_metric.${identityField.field}}}`, - }, - }; - - if (!identityField.field.includes('.')) { - return [setProcessor]; - } - - return [ - { - dot_expander: { - field: identityField.field, - path: `entity.identity.${identityField.field}.top_metric`, - }, - }, - setProcessor, - ]; - }) - .flat(); + return definition.identityFields.map((key) => ({ + set: { + if: `ctx.entity?.identity?.${key.field.replaceAll('.', '?.')} != null`, + field: key.field, + value: `{{entity.identity.${key.field}}}`, + }, + })); } function getCustomIngestPipelines(definition: EntityDefinition) { @@ -156,6 +139,55 @@ export function generateLatestProcessors(definition: EntityDefinition) { value: definition.identityFields.map((identityField) => identityField.field), }, }, + { + script: { + description: 'Generated the entity.id field', + source: cleanScript(` + // This function will recursively collect all the values of a HashMap of HashMaps + Collection collectValues(HashMap subject) { + Collection values = new ArrayList(); + // Iterate through the values + for(Object value: subject.values()) { + // If the value is a HashMap, recurse + if (value instanceof HashMap) { + values.addAll(collectValues((HashMap) value)); + } else { + values.add(String.valueOf(value)); + } + } + return values; + } + + // Create the string builder + StringBuilder entityId = new StringBuilder(); + + if (ctx["entity"]["identity"] != null) { + // Get the values as a collection + Collection values = collectValues(ctx["entity"]["identity"]); + + // Convert to a list and sort + List sortedValues = new ArrayList(values); + Collections.sort(sortedValues); + + // Create comma delimited string + for(String instanceValue: sortedValues) { + entityId.append(instanceValue); + entityId.append(":"); + } + + // Assign the entity.id + ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown"; + } + `), + }, + }, + { + fingerprint: { + fields: ['entity.id'], + target_field: 'entity.id', + method: 'MurmurHash3', + }, + }, ...(definition.staticFields != null ? Object.keys(definition.staticFields).map((field) => ({ set: { field, value: definition.staticFields![field] }, @@ -177,8 +209,8 @@ export function generateLatestProcessors(definition: EntityDefinition) { ignore_missing: true, }, }, + // This must happen AFTER we lift the identity fields into the root of the document { - // This must happen AFTER we lift the identity fields into the root of the document set: { field: 'entity.displayName', value: definition.displayNameTemplate, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts index 5cee21dc43a07..e07670c58fd9b 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts @@ -19,19 +19,23 @@ import { } from './install_entity_definition'; import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; import { - generateHistoryIndexTemplateId, - generateHistoryIngestPipelineId, - generateHistoryTransformId, generateLatestIndexTemplateId, generateLatestIngestPipelineId, generateLatestTransformId, } from './helpers/generate_component_id'; -import { generateHistoryTransform } from './transform/generate_history_transform'; import { generateLatestTransform } from './transform/generate_latest_transform'; import { entityDefinition as mockEntityDefinition } from './helpers/fixtures/entity_definition'; import { EntityDefinitionIdInvalid } from './errors/entity_definition_id_invalid'; import { EntityIdConflict } from './errors/entity_id_conflict_error'; +const getExpectedInstalledComponents = (definition: EntityDefinition) => { + return [ + { type: 'template', id: generateLatestIndexTemplateId(definition) }, + { type: 'ingest_pipeline', id: generateLatestIngestPipelineId(definition) }, + { type: 'transform', id: generateLatestTransformId(definition) }, + ]; +}; + const assertHasCreatedDefinition = ( definition: EntityDefinition, soClient: SavedObjectsClientContract, @@ -44,6 +48,7 @@ const assertHasCreatedDefinition = ( ...definition, installStatus: 'installing', installStartedAt: expect.any(String), + installedComponents: [], }, { id: definition.id, @@ -54,29 +59,17 @@ const assertHasCreatedDefinition = ( expect(soClient.update).toBeCalledTimes(1); expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, { installStatus: 'installed', + installedComponents: getExpectedInstalledComponents(definition), }); - expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2); - expect(esClient.indices.putIndexTemplate).toBeCalledWith( - expect.objectContaining({ - name: `entities_v1_history_${definition.id}_index_template`, - }) - ); + expect(esClient.indices.putIndexTemplate).toBeCalledTimes(1); expect(esClient.indices.putIndexTemplate).toBeCalledWith( expect.objectContaining({ name: `entities_v1_latest_${definition.id}_index_template`, }) ); - expect(esClient.ingest.putPipeline).toBeCalledTimes(2); - expect(esClient.ingest.putPipeline).toBeCalledWith({ - id: generateHistoryIngestPipelineId(definition), - processors: expect.anything(), - _meta: { - definitionVersion: definition.version, - managed: definition.managed, - }, - }); + expect(esClient.ingest.putPipeline).toBeCalledTimes(1); expect(esClient.ingest.putPipeline).toBeCalledWith({ id: generateLatestIngestPipelineId(definition), processors: expect.anything(), @@ -86,8 +79,7 @@ const assertHasCreatedDefinition = ( }, }); - expect(esClient.transform.putTransform).toBeCalledTimes(2); - expect(esClient.transform.putTransform).toBeCalledWith(generateHistoryTransform(definition)); + expect(esClient.transform.putTransform).toBeCalledTimes(1); expect(esClient.transform.putTransform).toBeCalledWith(generateLatestTransform(definition)); }; @@ -101,32 +93,21 @@ const assertHasUpgradedDefinition = ( ...definition, installStatus: 'upgrading', installStartedAt: expect.any(String), + installedComponents: getExpectedInstalledComponents(definition), }); expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, { installStatus: 'installed', + installedComponents: getExpectedInstalledComponents(definition), }); - expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2); - expect(esClient.indices.putIndexTemplate).toBeCalledWith( - expect.objectContaining({ - name: `entities_v1_history_${definition.id}_index_template`, - }) - ); + expect(esClient.indices.putIndexTemplate).toBeCalledTimes(1); expect(esClient.indices.putIndexTemplate).toBeCalledWith( expect.objectContaining({ name: `entities_v1_latest_${definition.id}_index_template`, }) ); - expect(esClient.ingest.putPipeline).toBeCalledTimes(2); - expect(esClient.ingest.putPipeline).toBeCalledWith({ - id: generateHistoryIngestPipelineId(definition), - processors: expect.anything(), - _meta: { - definitionVersion: definition.version, - managed: definition.managed, - }, - }); + expect(esClient.ingest.putPipeline).toBeCalledTimes(1); expect(esClient.ingest.putPipeline).toBeCalledWith({ id: generateLatestIngestPipelineId(definition), processors: expect.anything(), @@ -136,8 +117,7 @@ const assertHasUpgradedDefinition = ( }, }); - expect(esClient.transform.putTransform).toBeCalledTimes(2); - expect(esClient.transform.putTransform).toBeCalledWith(generateHistoryTransform(definition)); + expect(esClient.transform.putTransform).toBeCalledTimes(1); expect(esClient.transform.putTransform).toBeCalledWith(generateLatestTransform(definition)); }; @@ -148,13 +128,7 @@ const assertHasDeletedDefinition = ( ) => { assertHasDeletedTransforms(definition, esClient); - expect(esClient.ingest.deletePipeline).toBeCalledTimes(2); - expect(esClient.ingest.deletePipeline).toBeCalledWith( - { - id: generateHistoryIngestPipelineId(definition), - }, - { ignore: [404] } - ); + expect(esClient.ingest.deletePipeline).toBeCalledTimes(1); expect(esClient.ingest.deletePipeline).toBeCalledWith( { id: generateLatestIngestPipelineId(definition), @@ -162,13 +136,7 @@ const assertHasDeletedDefinition = ( { ignore: [404] } ); - expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2); - expect(esClient.indices.deleteIndexTemplate).toBeCalledWith( - { - name: generateHistoryIndexTemplateId(definition), - }, - { ignore: [404] } - ); + expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(1); expect(esClient.indices.deleteIndexTemplate).toBeCalledWith( { name: generateLatestIndexTemplateId(definition), @@ -184,33 +152,21 @@ const assertHasDeletedTransforms = ( definition: EntityDefinition, esClient: ElasticsearchClient ) => { - expect(esClient.transform.stopTransform).toBeCalledTimes(2); - expect(esClient.transform.stopTransform).toBeCalledWith( - expect.objectContaining({ - transform_id: generateHistoryTransformId(definition), - }), - expect.anything() - ); - expect(esClient.transform.deleteTransform).toBeCalledWith( - expect.objectContaining({ - transform_id: generateHistoryTransformId(definition), - }), - expect.anything() - ); + expect(esClient.transform.stopTransform).toBeCalledTimes(1); expect(esClient.transform.stopTransform).toBeCalledWith( expect.objectContaining({ transform_id: generateLatestTransformId(definition), }), expect.anything() ); + + expect(esClient.transform.deleteTransform).toBeCalledTimes(1); expect(esClient.transform.deleteTransform).toBeCalledWith( expect.objectContaining({ transform_id: generateLatestTransformId(definition), }), expect.anything() ); - - expect(esClient.transform.deleteTransform).toBeCalledTimes(2); }; describe('install_entity_definition', () => { @@ -223,7 +179,7 @@ describe('install_entity_definition', () => { installEntityDefinition({ esClient, soClient, - definition: { id: 'a'.repeat(40) } as EntityDefinition, + definition: { id: 'a'.repeat(50) } as EntityDefinition, logger: loggerMock.create(), }) ).rejects.toThrow(EntityDefinitionIdInvalid); @@ -242,6 +198,7 @@ describe('install_entity_definition', () => { attributes: { ...mockEntityDefinition, installStatus: 'installed', + installedComponents: [], }, }, ], @@ -264,6 +221,12 @@ describe('install_entity_definition', () => { const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; const soClient = savedObjectsClientMock.create(); soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installEntityDefinition({ esClient, @@ -300,6 +263,12 @@ describe('install_entity_definition', () => { const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; const soClient = savedObjectsClientMock.create(); soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, @@ -329,6 +298,7 @@ describe('install_entity_definition', () => { attributes: { ...mockEntityDefinition, installStatus: 'installed', + installedComponents: getExpectedInstalledComponents(mockEntityDefinition), }, }, ], @@ -336,6 +306,12 @@ describe('install_entity_definition', () => { page: 1, per_page: 10, }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, @@ -367,6 +343,7 @@ describe('install_entity_definition', () => { attributes: { ...mockEntityDefinition, installStatus: 'installed', + installedComponents: getExpectedInstalledComponents(mockEntityDefinition), }, }, ], @@ -374,6 +351,12 @@ describe('install_entity_definition', () => { page: 1, per_page: 10, }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, @@ -407,6 +390,7 @@ describe('install_entity_definition', () => { // upgrading for 1h installStatus: 'upgrading', installStartedAt: moment().subtract(1, 'hour').toISOString(), + installedComponents: getExpectedInstalledComponents(mockEntityDefinition), }, }, ], @@ -414,6 +398,12 @@ describe('install_entity_definition', () => { page: 1, per_page: 10, }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, @@ -442,6 +432,7 @@ describe('install_entity_definition', () => { ...mockEntityDefinition, installStatus: 'failed', installStartedAt: new Date().toISOString(), + installedComponents: getExpectedInstalledComponents(mockEntityDefinition), }, }, ], @@ -449,6 +440,12 @@ describe('install_entity_definition', () => { page: 1, per_page: 10, }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts index 7d6dee4fb2ced..b4adedaf10374 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts @@ -10,39 +10,25 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; -import { - generateHistoryIndexTemplateId, - generateLatestIndexTemplateId, -} from './helpers/generate_component_id'; -import { - createAndInstallHistoryIngestPipeline, - createAndInstallLatestIngestPipeline, -} from './create_and_install_ingest_pipeline'; -import { - createAndInstallHistoryBackfillTransform, - createAndInstallHistoryTransform, - createAndInstallLatestTransform, -} from './create_and_install_transform'; +import { generateLatestIndexTemplateId } from './helpers/generate_component_id'; +import { createAndInstallIngestPipelines } from './create_and_install_ingest_pipeline'; +import { createAndInstallTransforms } from './create_and_install_transform'; import { validateDefinitionCanCreateValidTransformIds } from './transform/validate_transform_ids'; import { deleteEntityDefinition } from './delete_entity_definition'; -import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline'; +import { deleteLatestIngestPipeline } from './delete_ingest_pipeline'; import { findEntityDefinitionById } from './find_entity_definition'; import { entityDefinitionExists, saveEntityDefinition, updateEntityDefinition, } from './save_entity_definition'; - -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; -import { deleteTemplate, upsertTemplate } from '../manage_index_templates'; -import { generateEntitiesLatestIndexTemplateConfig } from './templates/entities_latest_template'; -import { generateEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template'; +import { createAndInstallTemplates, deleteTemplate } from '../manage_index_templates'; import { EntityIdConflict } from './errors/entity_id_conflict_error'; import { EntityDefinitionNotFound } from './errors/entity_not_found'; import { mergeEntityDefinitionUpdate } from './helpers/merge_definition_update'; import { EntityDefinitionWithState } from './types'; -import { stopTransforms } from './stop_transforms'; -import { deleteTransforms } from './delete_transforms'; +import { stopLatestTransform, stopTransforms } from './stop_transforms'; +import { deleteLatestTransform, deleteTransforms } from './delete_transforms'; export interface InstallDefinitionParams { esClient: ElasticsearchClient; @@ -51,16 +37,6 @@ export interface InstallDefinitionParams { logger: Logger; } -const throwIfRejected = (values: Array | PromiseRejectedResult>) => { - const rejectedPromise = values.find( - (value) => value.status === 'rejected' - ) as PromiseRejectedResult; - if (rejectedPromise) { - throw new Error(rejectedPromise.reason); - } - return values; -}; - // install an entity definition from scratch with all its required components // after verifying that the definition id is valid and available. // attempt to remove all installed components if the installation fails. @@ -72,42 +48,35 @@ export async function installEntityDefinition({ }: InstallDefinitionParams): Promise { validateDefinitionCanCreateValidTransformIds(definition); - try { - if (await entityDefinitionExists(soClient, definition.id)) { - throw new EntityIdConflict( - `Entity definition with [${definition.id}] already exists.`, - definition - ); - } + if (await entityDefinitionExists(soClient, definition.id)) { + throw new EntityIdConflict( + `Entity definition with [${definition.id}] already exists.`, + definition + ); + } + try { const entityDefinition = await saveEntityDefinition(soClient, { ...definition, installStatus: 'installing', installStartedAt: new Date().toISOString(), + installedComponents: [], }); return await install({ esClient, soClient, logger, definition: entityDefinition }); } catch (e) { logger.error(`Failed to install entity definition ${definition.id}: ${e}`); - await stopAndDeleteTransforms(esClient, definition, logger); - await Promise.all([ - deleteHistoryIngestPipeline(esClient, definition, logger), - deleteLatestIngestPipeline(esClient, definition, logger), - ]); + await stopLatestTransform(esClient, definition, logger); + await deleteLatestTransform(esClient, definition, logger); - await Promise.all([ - deleteTemplate({ - esClient, - logger, - name: generateHistoryIndexTemplateId(definition), - }), - deleteTemplate({ - esClient, - logger, - name: generateLatestIndexTemplateId(definition), - }), - ]); + await deleteLatestIngestPipeline(esClient, definition, logger); + + await deleteTemplate({ + esClient, + logger, + name: generateLatestIndexTemplateId(definition), + }); await deleteEntityDefinition(soClient, definition).catch((err) => { if (err instanceof EntityDefinitionNotFound) { @@ -191,36 +160,19 @@ async function install({ ); logger.debug(`Installing index templates for definition ${definition.id}`); - await Promise.allSettled([ - upsertTemplate({ - esClient, - logger, - template: generateEntitiesHistoryIndexTemplateConfig(definition), - }), - upsertTemplate({ - esClient, - logger, - template: generateEntitiesLatestIndexTemplateConfig(definition), - }), - ]).then(throwIfRejected); + const templates = await createAndInstallTemplates(esClient, definition, logger); logger.debug(`Installing ingest pipelines for definition ${definition.id}`); - await Promise.allSettled([ - createAndInstallHistoryIngestPipeline(esClient, definition, logger), - createAndInstallLatestIngestPipeline(esClient, definition, logger), - ]).then(throwIfRejected); + const pipelines = await createAndInstallIngestPipelines(esClient, definition, logger); logger.debug(`Installing transforms for definition ${definition.id}`); - await Promise.allSettled([ - createAndInstallHistoryTransform(esClient, definition, logger), - isBackfillEnabled(definition) - ? createAndInstallHistoryBackfillTransform(esClient, definition, logger) - : Promise.resolve(), - createAndInstallLatestTransform(esClient, definition, logger), - ]).then(throwIfRejected); - - await updateEntityDefinition(soClient, definition.id, { installStatus: 'installed' }); - return { ...definition, installStatus: 'installed' }; + const transforms = await createAndInstallTransforms(esClient, definition, logger); + + const updatedProps = await updateEntityDefinition(soClient, definition.id, { + installStatus: 'installed', + installedComponents: [...templates, ...pipelines, ...transforms], + }); + return { ...definition, ...updatedProps.attributes }; } // stop and delete the current transforms and reinstall all the components diff --git a/x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts index 2dff5178aeeaf..d32edfa146917 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts @@ -41,5 +41,5 @@ export async function updateEntityDefinition( id: string, definition: Partial ) { - await soClient.update(SO_ENTITY_DEFINITION_TYPE, id, definition); + return await soClient.update(SO_ENTITY_DEFINITION_TYPE, id, definition); } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts b/x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts index ea2ec7adb5ddc..f4cd8fc89dd11 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts @@ -7,13 +7,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { - generateHistoryBackfillTransformId, - generateHistoryTransformId, - generateLatestTransformId, -} from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; export async function startTransforms( esClient: ElasticsearchClient, @@ -21,28 +15,15 @@ export async function startTransforms( logger: Logger ) { try { - const historyTransformId = generateHistoryTransformId(definition); - const latestTransformId = generateLatestTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.startTransform({ transform_id: historyTransformId }, { ignore: [409] }), - { logger } - ); - if (isBackfillEnabled(definition)) { - const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.startTransform( - { transform_id: historyBackfillTransformId }, - { ignore: [409] } - ), - { logger } - ); - } - await retryTransientEsErrors( - () => - esClient.transform.startTransform({ transform_id: latestTransformId }, { ignore: [409] }), - { logger } + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'transform') + .map(({ id }) => + retryTransientEsErrors( + () => esClient.transform.startTransform({ transform_id: id }, { ignore: [409] }), + { logger } + ) + ) ); } catch (err) { logger.error(`Cannot start entity transforms [${definition.id}]: ${err}`); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts b/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts index 98f9ad351e377..9aabad926b239 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts @@ -8,14 +8,8 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { - generateHistoryTransformId, - generateHistoryBackfillTransformId, - generateLatestTransformId, -} from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; - -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; +import { generateLatestTransformId } from './helpers/generate_component_id'; export async function stopTransforms( esClient: ElasticsearchClient, @@ -23,43 +17,46 @@ export async function stopTransforms( logger: Logger ) { try { - const historyTransformId = generateHistoryTransformId(definition); - const latestTransformId = generateLatestTransformId(definition); - - await retryTransientEsErrors( - () => - esClient.transform.stopTransform( - { transform_id: historyTransformId, wait_for_completion: true, force: true }, - { ignore: [409, 404] } - ), - { logger } + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'transform') + .map(({ id }) => + retryTransientEsErrors( + () => + esClient.transform.stopTransform( + { transform_id: id, wait_for_completion: true, force: true }, + { ignore: [409, 404] } + ), + { logger } + ) + ) ); + } catch (e) { + logger.error(`Cannot stop transforms for definition [${definition.id}]: ${e}`); + throw e; + } +} - if (isBackfillEnabled(definition)) { - const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.stopTransform( - { - transform_id: historyBackfillTransformId, - wait_for_completion: true, - force: true, - }, - { ignore: [409, 404] } - ), - { logger } - ); - } +export async function stopLatestTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { await retryTransientEsErrors( () => esClient.transform.stopTransform( - { transform_id: latestTransformId, wait_for_completion: true, force: true }, + { + transform_id: generateLatestTransformId(definition), + wait_for_completion: true, + force: true, + }, { ignore: [409, 404] } ), { logger } ); } catch (e) { - logger.error(`Cannot stop entity transforms [${definition.id}]: ${e}`); + logger.error(`Cannot stop latest transform for definition [${definition.id}]: ${e}`); throw e; } } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap deleted file mode 100644 index fd4ed11f8cb94..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap +++ /dev/null @@ -1,152 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`generateEntitiesHistoryIndexTemplateConfig(definition) should generate a valid index template for builtin definition 1`] = ` -Object { - "_meta": Object { - "description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", - "ecs_version": "8.0.0", - "managed": true, - "managed_by": "elastic_entity_model", - }, - "composed_of": Array [ - "entities_v1_history_base", - "entities_v1_entity", - "entities_v1_event", - ], - "ignore_missing_component_templates": Array [], - "index_patterns": Array [ - ".entities.v1.history.builtin_mock_entity_definition.*", - ], - "name": "entities_v1_history_builtin_mock_entity_definition_index_template", - "priority": 200, - "template": Object { - "aliases": Object { - "entities-service-history": Object {}, - }, - "mappings": Object { - "_meta": Object { - "version": "1.6.0", - }, - "date_detection": false, - "dynamic_templates": Array [ - Object { - "strings_as_keyword": Object { - "mapping": Object { - "fields": Object { - "text": Object { - "type": "text", - }, - }, - "ignore_above": 1024, - "type": "keyword", - }, - "match_mapping_type": "string", - }, - }, - Object { - "entity_metrics": Object { - "mapping": Object { - "type": "{dynamic_type}", - }, - "match_mapping_type": Array [ - "long", - "double", - ], - "path_match": "entity.metrics.*", - }, - }, - ], - }, - "settings": Object { - "index": Object { - "codec": "best_compression", - "mapping": Object { - "total_fields": Object { - "limit": 2000, - }, - }, - }, - }, - }, -} -`; - -exports[`generateEntitiesHistoryIndexTemplateConfig(definition) should generate a valid index template for custom definition 1`] = ` -Object { - "_meta": Object { - "description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", - "ecs_version": "8.0.0", - "managed": true, - "managed_by": "elastic_entity_model", - }, - "composed_of": Array [ - "entities_v1_history_base", - "entities_v1_entity", - "entities_v1_event", - "admin-console-services@platform", - "admin-console-services-history@platform", - "admin-console-services@custom", - "admin-console-services-history@custom", - ], - "ignore_missing_component_templates": Array [ - "admin-console-services@platform", - "admin-console-services-history@platform", - "admin-console-services@custom", - "admin-console-services-history@custom", - ], - "index_patterns": Array [ - ".entities.v1.history.admin-console-services.*", - ], - "name": "entities_v1_history_admin-console-services_index_template", - "priority": 200, - "template": Object { - "aliases": Object { - "entities-service-history": Object {}, - }, - "mappings": Object { - "_meta": Object { - "version": "1.6.0", - }, - "date_detection": false, - "dynamic_templates": Array [ - Object { - "strings_as_keyword": Object { - "mapping": Object { - "fields": Object { - "text": Object { - "type": "text", - }, - }, - "ignore_above": 1024, - "type": "keyword", - }, - "match_mapping_type": "string", - }, - }, - Object { - "entity_metrics": Object { - "mapping": Object { - "type": "{dynamic_type}", - }, - "match_mapping_type": Array [ - "long", - "double", - ], - "path_match": "entity.metrics.*", - }, - }, - ], - }, - "settings": Object { - "index": Object { - "codec": "best_compression", - "mapping": Object { - "total_fields": Object { - "limit": 2000, - }, - }, - }, - }, - }, -} -`; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.test.ts deleted file mode 100644 index 72e8d8591ab2d..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entityDefinition, builtInEntityDefinition } from '../helpers/fixtures'; -import { generateEntitiesHistoryIndexTemplateConfig } from './entities_history_template'; - -describe('generateEntitiesHistoryIndexTemplateConfig(definition)', () => { - it('should generate a valid index template for custom definition', () => { - const template = generateEntitiesHistoryIndexTemplateConfig(entityDefinition); - expect(template).toMatchSnapshot(); - }); - - it('should generate a valid index template for builtin definition', () => { - const template = generateEntitiesHistoryIndexTemplateConfig(builtInEntityDefinition); - expect(template).toMatchSnapshot(); - }); -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.ts b/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.ts deleted file mode 100644 index b1539d8108a6d..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { - ENTITY_HISTORY, - EntityDefinition, - entitiesIndexPattern, - entitiesAliasPattern, - ENTITY_SCHEMA_VERSION_V1, -} from '@kbn/entities-schema'; -import { generateHistoryIndexTemplateId } from '../helpers/generate_component_id'; -import { - ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, - ENTITY_EVENT_COMPONENT_TEMPLATE_V1, - ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1, -} from '../../../../common/constants_entities'; -import { getCustomHistoryTemplateComponents } from '../../../templates/components/helpers'; - -export const generateEntitiesHistoryIndexTemplateConfig = ( - definition: EntityDefinition -): IndicesPutIndexTemplateRequest => ({ - name: generateHistoryIndexTemplateId(definition), - _meta: { - description: - "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", - ecs_version: '8.0.0', - managed: true, - managed_by: 'elastic_entity_model', - }, - ignore_missing_component_templates: getCustomHistoryTemplateComponents(definition), - composed_of: [ - ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1, - ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, - ENTITY_EVENT_COMPONENT_TEMPLATE_V1, - ...getCustomHistoryTemplateComponents(definition), - ], - index_patterns: [ - `${entitiesIndexPattern({ - schemaVersion: ENTITY_SCHEMA_VERSION_V1, - dataset: ENTITY_HISTORY, - definitionId: definition.id, - })}.*`, - ], - priority: 200, - template: { - aliases: { - [entitiesAliasPattern({ type: definition.type, dataset: ENTITY_HISTORY })]: {}, - }, - mappings: { - _meta: { - version: '1.6.0', - }, - date_detection: false, - dynamic_templates: [ - { - strings_as_keyword: { - mapping: { - ignore_above: 1024, - type: 'keyword', - fields: { - text: { - type: 'text', - }, - }, - }, - match_mapping_type: 'string', - }, - }, - { - entity_metrics: { - mapping: { - type: '{dynamic_type}', - }, - match_mapping_type: ['long', 'double'], - path_match: 'entity.metrics.*', - }, - }, - ], - }, - settings: { - index: { - codec: 'best_compression', - mapping: { - total_fields: { - limit: 2000, - }, - }, - }, - }, - }, -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts b/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts index ea476cf769644..e0c02c7471217 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts @@ -19,7 +19,7 @@ import { ENTITY_EVENT_COMPONENT_TEMPLATE_V1, ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1, } from '../../../../common/constants_entities'; -import { getCustomLatestTemplateComponents } from '../../../templates/components/helpers'; +import { isBuiltinDefinition } from '../helpers/is_builtin_definition'; export const generateEntitiesLatestIndexTemplateConfig = ( definition: EntityDefinition @@ -94,3 +94,16 @@ export const generateEntitiesLatestIndexTemplateConfig = ( }, }, }); + +function getCustomLatestTemplateComponents(definition: EntityDefinition) { + if (isBuiltinDefinition(definition)) { + return []; + } + + return [ + `${definition.id}@platform`, // @platform goes before so it can be overwritten by custom + `${definition.id}-latest@platform`, + `${definition.id}@custom`, + `${definition.id}-latest@custom`, + ]; +} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap deleted file mode 100644 index b19a805b24b12..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap +++ /dev/null @@ -1,305 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`generateHistoryTransform(definition) should generate a valid history backfill transform 1`] = ` -Object { - "_meta": Object { - "definitionVersion": "999.999.999", - "managed": false, - }, - "defer_validation": true, - "dest": Object { - "index": ".entities.v1.history.noop", - "pipeline": "entities-v1-history-admin-console-services-backfill", - }, - "frequency": "5m", - "pivot": Object { - "aggs": Object { - "_errorRate_A": Object { - "filter": Object { - "bool": Object { - "minimum_should_match": 1, - "should": Array [ - Object { - "match_phrase": Object { - "log.level": "ERROR", - }, - }, - ], - }, - }, - }, - "_logRate_A": Object { - "filter": Object { - "bool": Object { - "minimum_should_match": 1, - "should": Array [ - Object { - "exists": Object { - "field": "log.level", - }, - }, - ], - }, - }, - }, - "entity.lastSeenTimestamp": Object { - "max": Object { - "field": "@timestamp", - }, - }, - "entity.metadata.host.name": Object { - "terms": Object { - "field": "host.name", - "size": 1000, - }, - }, - "entity.metadata.host.os.name": Object { - "terms": Object { - "field": "host.os.name", - "size": 1000, - }, - }, - "entity.metadata.sourceIndex": Object { - "terms": Object { - "field": "_index", - "size": 1000, - }, - }, - "entity.metadata.tags": Object { - "terms": Object { - "field": "tags", - "size": 1000, - }, - }, - "entity.metrics.errorRate": Object { - "bucket_script": Object { - "buckets_path": Object { - "A": "_errorRate_A>_count", - }, - "script": Object { - "lang": "painless", - "source": "params.A", - }, - }, - }, - "entity.metrics.logRate": Object { - "bucket_script": Object { - "buckets_path": Object { - "A": "_logRate_A>_count", - }, - "script": Object { - "lang": "painless", - "source": "params.A", - }, - }, - }, - }, - "group_by": Object { - "@timestamp": Object { - "date_histogram": Object { - "field": "@timestamp", - "fixed_interval": "1m", - }, - }, - "entity.identity.event.category": Object { - "terms": Object { - "field": "event.category", - "missing_bucket": true, - }, - }, - "entity.identity.log.logger": Object { - "terms": Object { - "field": "log.logger", - "missing_bucket": false, - }, - }, - }, - }, - "settings": Object { - "deduce_mappings": false, - "unattended": true, - }, - "source": Object { - "index": Array [ - "kbn-data-forge-fake_stack.*", - ], - "query": Object { - "bool": Object { - "filter": Array [ - Object { - "range": Object { - "@timestamp": Object { - "gte": "now-72h", - }, - }, - }, - Object { - "exists": Object { - "field": "log.logger", - }, - }, - ], - }, - }, - }, - "sync": Object { - "time": Object { - "delay": "15m", - "field": "@timestamp", - }, - }, - "transform_id": "entities-v1-history-backfill-admin-console-services-backfill", -} -`; - -exports[`generateHistoryTransform(definition) should generate a valid history transform 1`] = ` -Object { - "_meta": Object { - "definitionVersion": "1.0.0", - "managed": false, - }, - "defer_validation": true, - "dest": Object { - "index": ".entities.v1.history.noop", - "pipeline": "entities-v1-history-admin-console-services", - }, - "frequency": "2m", - "pivot": Object { - "aggs": Object { - "_errorRate_A": Object { - "filter": Object { - "bool": Object { - "minimum_should_match": 1, - "should": Array [ - Object { - "match_phrase": Object { - "log.level": "ERROR", - }, - }, - ], - }, - }, - }, - "_logRate_A": Object { - "filter": Object { - "bool": Object { - "minimum_should_match": 1, - "should": Array [ - Object { - "exists": Object { - "field": "log.level", - }, - }, - ], - }, - }, - }, - "entity.lastSeenTimestamp": Object { - "max": Object { - "field": "@timestamp", - }, - }, - "entity.metadata.host.name": Object { - "terms": Object { - "field": "host.name", - "size": 1000, - }, - }, - "entity.metadata.host.os.name": Object { - "terms": Object { - "field": "host.os.name", - "size": 1000, - }, - }, - "entity.metadata.sourceIndex": Object { - "terms": Object { - "field": "_index", - "size": 1000, - }, - }, - "entity.metadata.tags": Object { - "terms": Object { - "field": "tags", - "size": 1000, - }, - }, - "entity.metrics.errorRate": Object { - "bucket_script": Object { - "buckets_path": Object { - "A": "_errorRate_A>_count", - }, - "script": Object { - "lang": "painless", - "source": "params.A", - }, - }, - }, - "entity.metrics.logRate": Object { - "bucket_script": Object { - "buckets_path": Object { - "A": "_logRate_A>_count", - }, - "script": Object { - "lang": "painless", - "source": "params.A", - }, - }, - }, - }, - "group_by": Object { - "@timestamp": Object { - "date_histogram": Object { - "field": "@timestamp", - "fixed_interval": "1m", - }, - }, - "entity.identity.event.category": Object { - "terms": Object { - "field": "event.category", - "missing_bucket": true, - }, - }, - "entity.identity.log.logger": Object { - "terms": Object { - "field": "log.logger", - "missing_bucket": false, - }, - }, - }, - }, - "settings": Object { - "deduce_mappings": false, - "unattended": true, - }, - "source": Object { - "index": Array [ - "kbn-data-forge-fake_stack.*", - ], - "query": Object { - "bool": Object { - "filter": Array [ - Object { - "exists": Object { - "field": "log.logger", - }, - }, - Object { - "range": Object { - "@timestamp": Object { - "gte": "now-10m", - }, - }, - }, - ], - }, - }, - }, - "sync": Object { - "time": Object { - "delay": "2m", - "field": "@timestamp", - }, - }, - "transform_id": "entities-v1-history-admin-console-services", -} -`; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap index ab1224525f4d7..49f8ff4536120 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap @@ -14,76 +14,37 @@ Object { "frequency": "30s", "pivot": Object { "aggs": Object { - "_errorRate": Object { - "top_metrics": Object { - "metrics": Array [ - Object { - "field": "entity.metrics.errorRate", - }, - ], - "sort": Array [ - Object { - "@timestamp": "desc", - }, - ], - }, - }, - "_logRate": Object { - "top_metrics": Object { - "metrics": Array [ - Object { - "field": "entity.metrics.logRate", - }, - ], - "sort": Array [ - Object { - "@timestamp": "desc", - }, - ], - }, - }, - "entity.firstSeenTimestamp": Object { - "min": Object { - "field": "@timestamp", - }, - }, - "entity.identity.event.category": Object { - "aggs": Object { - "top_metric": Object { - "top_metrics": Object { - "metrics": Object { - "field": "event.category", - }, - "sort": "_score", - }, - }, - }, + "_errorRate_A": Object { "filter": Object { - "exists": Object { - "field": "event.category", - }, - }, - }, - "entity.identity.log.logger": Object { - "aggs": Object { - "top_metric": Object { - "top_metrics": Object { - "metrics": Object { - "field": "log.logger", + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "log.level": "ERROR", + }, }, - "sort": "_score", - }, + ], }, }, + }, + "_logRate_A": Object { "filter": Object { - "exists": Object { - "field": "log.logger", + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "exists": Object { + "field": "log.level", + }, + }, + ], }, }, }, "entity.lastSeenTimestamp": Object { "max": Object { - "field": "entity.lastSeenTimestamp", + "field": "@timestamp", }, }, "entity.metadata.host.name": Object { @@ -91,14 +52,14 @@ Object { "data": Object { "terms": Object { "field": "host.name", - "size": 1000, + "size": 10, }, }, }, "filter": Object { "range": Object { "@timestamp": Object { - "gte": "now-360s", + "gte": "now-10m", }, }, }, @@ -108,14 +69,14 @@ Object { "data": Object { "terms": Object { "field": "host.os.name", - "size": 1000, + "size": 10, }, }, }, "filter": Object { "range": Object { "@timestamp": Object { - "gte": "now-360s", + "gte": "now-10m", }, }, }, @@ -124,15 +85,15 @@ Object { "aggs": Object { "data": Object { "terms": Object { - "field": "sourceIndex", - "size": 1000, + "field": "_index", + "size": 10, }, }, }, "filter": Object { "range": Object { "@timestamp": Object { - "gte": "now-360s", + "gte": "now-10m", }, }, }, @@ -142,14 +103,14 @@ Object { "data": Object { "terms": Object { "field": "tags", - "size": 1000, + "size": 10, }, }, }, "filter": Object { "range": Object { "@timestamp": Object { - "gte": "now-360s", + "gte": "now-10m", }, }, }, @@ -157,24 +118,37 @@ Object { "entity.metrics.errorRate": Object { "bucket_script": Object { "buckets_path": Object { - "value": "_errorRate[entity.metrics.errorRate]", + "A": "_errorRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A", }, - "script": "params.value", }, }, "entity.metrics.logRate": Object { "bucket_script": Object { "buckets_path": Object { - "value": "_logRate[entity.metrics.logRate]", + "A": "_logRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A", }, - "script": "params.value", }, }, }, "group_by": Object { - "entity.id": Object { + "entity.identity.event.category": Object { + "terms": Object { + "field": "event.category", + "missing_bucket": true, + }, + }, + "entity.identity.log.logger": Object { "terms": Object { - "field": "entity.id", + "field": "log.logger", + "missing_bucket": false, }, }, }, @@ -184,12 +158,32 @@ Object { "unattended": true, }, "source": Object { - "index": ".entities.v1.history.admin-console-services.*", + "index": Array [ + "kbn-data-forge-fake_stack.*", + ], + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "exists": Object { + "field": "log.logger", + }, + }, + Object { + "range": Object { + "@timestamp": Object { + "gte": "now-10m", + }, + }, + }, + ], + }, + }, }, "sync": Object { "time": Object { - "delay": "1s", - "field": "event.ingested", + "delay": "10s", + "field": "@timestamp", }, }, "transform_id": "entities-v1-latest-admin-console-services", diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts deleted file mode 100644 index f49ec0cd88a37..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entityDefinition } from '../helpers/fixtures/entity_definition'; -import { entityDefinitionWithBackfill } from '../helpers/fixtures/entity_definition_with_backfill'; -import { - generateBackfillHistoryTransform, - generateHistoryTransform, -} from './generate_history_transform'; - -describe('generateHistoryTransform(definition)', () => { - it('should generate a valid history transform', () => { - const transform = generateHistoryTransform(entityDefinition); - expect(transform).toMatchSnapshot(); - }); - it('should generate a valid history backfill transform', () => { - const transform = generateBackfillHistoryTransform(entityDefinitionWithBackfill); - expect(transform).toMatchSnapshot(); - }); -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts deleted file mode 100644 index 239359738624c..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; -import { - QueryDslQueryContainer, - TransformPutTransformRequest, -} from '@elastic/elasticsearch/lib/api/types'; -import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; -import { generateHistoryMetricAggregations } from './generate_metric_aggregations'; -import { - ENTITY_DEFAULT_HISTORY_FREQUENCY, - ENTITY_DEFAULT_HISTORY_SYNC_DELAY, -} from '../../../../common/constants_entities'; -import { generateHistoryMetadataAggregations } from './generate_metadata_aggregations'; -import { - generateHistoryTransformId, - generateHistoryIngestPipelineId, - generateHistoryIndexName, - generateHistoryBackfillTransformId, -} from '../helpers/generate_component_id'; -import { isBackfillEnabled } from '../helpers/is_backfill_enabled'; - -export function generateHistoryTransform( - definition: EntityDefinition -): TransformPutTransformRequest { - const filter: QueryDslQueryContainer[] = []; - - if (definition.filter) { - filter.push(getElasticsearchQueryOrThrow(definition.filter)); - } - - if (definition.identityFields.some(({ optional }) => !optional)) { - definition.identityFields - .filter(({ optional }) => !optional) - .forEach(({ field }) => { - filter.push({ exists: { field } }); - }); - } - - filter.push({ - range: { - [definition.history.timestampField]: { - gte: `now-${definition.history.settings.lookbackPeriod}`, - }, - }, - }); - - return generateTransformPutRequest({ - definition, - filter, - transformId: generateHistoryTransformId(definition), - frequency: definition.history.settings.frequency, - syncDelay: definition.history.settings.syncDelay, - }); -} - -export function generateBackfillHistoryTransform( - definition: EntityDefinition -): TransformPutTransformRequest { - if (!isBackfillEnabled(definition)) { - throw new Error( - 'generateBackfillHistoryTransform called without history.settings.backfillSyncDelay set' - ); - } - - const filter: QueryDslQueryContainer[] = []; - - if (definition.filter) { - filter.push(getElasticsearchQueryOrThrow(definition.filter)); - } - - if (definition.history.settings.backfillLookbackPeriod) { - filter.push({ - range: { - [definition.history.timestampField]: { - gte: `now-${definition.history.settings.backfillLookbackPeriod}`, - }, - }, - }); - } - - if (definition.identityFields.some(({ optional }) => !optional)) { - definition.identityFields - .filter(({ optional }) => !optional) - .forEach(({ field }) => { - filter.push({ exists: { field } }); - }); - } - - return generateTransformPutRequest({ - definition, - filter, - transformId: generateHistoryBackfillTransformId(definition), - frequency: definition.history.settings.backfillFrequency, - syncDelay: definition.history.settings.backfillSyncDelay, - }); -} - -const generateTransformPutRequest = ({ - definition, - filter, - transformId, - frequency, - syncDelay, -}: { - definition: EntityDefinition; - transformId: string; - filter: QueryDslQueryContainer[]; - frequency?: string; - syncDelay?: string; -}) => { - return { - transform_id: transformId, - _meta: { - definitionVersion: definition.version, - managed: definition.managed, - }, - defer_validation: true, - source: { - index: definition.indexPatterns, - ...(filter.length > 0 && { - query: { - bool: { - filter, - }, - }, - }), - }, - dest: { - index: `${generateHistoryIndexName({ id: 'noop' } as EntityDefinition)}`, - pipeline: generateHistoryIngestPipelineId(definition), - }, - frequency: frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY, - sync: { - time: { - field: definition.history.settings.syncField || definition.history.timestampField, - delay: syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY, - }, - }, - settings: { - deduce_mappings: false, - unattended: true, - }, - pivot: { - group_by: { - ...definition.identityFields.reduce( - (acc, id) => ({ - ...acc, - [`entity.identity.${id.field}`]: { - terms: { field: id.field, missing_bucket: id.optional }, - }, - }), - {} - ), - ['@timestamp']: { - date_histogram: { - field: definition.history.timestampField, - fixed_interval: definition.history.interval, - }, - }, - }, - aggs: { - ...generateHistoryMetricAggregations(definition), - ...generateHistoryMetadataAggregations(definition), - 'entity.lastSeenTimestamp': { - max: { - field: definition.history.timestampField, - }, - }, - }, - }, - }; -}; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts index 85ee57fefea2c..573bb2225f183 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts @@ -5,44 +5,97 @@ * 2.0. */ -import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; import { EntityDefinition } from '@kbn/entities-schema'; +import { + QueryDslQueryContainer, + TransformPutTransformRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; +import { generateLatestMetricAggregations } from './generate_metric_aggregations'; import { ENTITY_DEFAULT_LATEST_FREQUENCY, ENTITY_DEFAULT_LATEST_SYNC_DELAY, } from '../../../../common/constants_entities'; import { - generateHistoryIndexName, - generateLatestIndexName, - generateLatestIngestPipelineId, generateLatestTransformId, + generateLatestIngestPipelineId, + generateLatestIndexName, } from '../helpers/generate_component_id'; -import { generateIdentityAggregations } from './generate_identity_aggregations'; import { generateLatestMetadataAggregations } from './generate_metadata_aggregations'; -import { generateLatestMetricAggregations } from './generate_metric_aggregations'; export function generateLatestTransform( definition: EntityDefinition ): TransformPutTransformRequest { + const filter: QueryDslQueryContainer[] = []; + + if (definition.filter) { + filter.push(getElasticsearchQueryOrThrow(definition.filter)); + } + + if (definition.identityFields.some(({ optional }) => !optional)) { + definition.identityFields + .filter(({ optional }) => !optional) + .forEach(({ field }) => { + filter.push({ exists: { field } }); + }); + } + + filter.push({ + range: { + [definition.latest.timestampField]: { + gte: `now-${definition.latest.lookbackPeriod}`, + }, + }, + }); + + return generateTransformPutRequest({ + definition, + filter, + transformId: generateLatestTransformId(definition), + frequency: definition.latest.settings?.frequency ?? ENTITY_DEFAULT_LATEST_FREQUENCY, + syncDelay: definition.latest.settings?.syncDelay ?? ENTITY_DEFAULT_LATEST_SYNC_DELAY, + }); +} + +const generateTransformPutRequest = ({ + definition, + filter, + transformId, + frequency, + syncDelay, +}: { + definition: EntityDefinition; + transformId: string; + filter: QueryDslQueryContainer[]; + frequency: string; + syncDelay: string; +}) => { return { - transform_id: generateLatestTransformId(definition), + transform_id: transformId, _meta: { definitionVersion: definition.version, managed: definition.managed, }, defer_validation: true, source: { - index: `${generateHistoryIndexName(definition)}.*`, + index: definition.indexPatterns, + ...(filter.length > 0 && { + query: { + bool: { + filter, + }, + }, + }), }, dest: { index: `${generateLatestIndexName({ id: 'noop' } as EntityDefinition)}`, pipeline: generateLatestIngestPipelineId(definition), }, - frequency: definition.latest?.settings?.frequency ?? ENTITY_DEFAULT_LATEST_FREQUENCY, + frequency, sync: { time: { - field: definition.latest?.settings?.syncField ?? 'event.ingested', - delay: definition.latest?.settings?.syncDelay ?? ENTITY_DEFAULT_LATEST_SYNC_DELAY, + field: definition.latest.settings?.syncField || definition.latest.timestampField, + delay: syncDelay, }, }, settings: { @@ -51,25 +104,25 @@ export function generateLatestTransform( }, pivot: { group_by: { - ['entity.id']: { - terms: { field: 'entity.id' }, - }, + ...definition.identityFields.reduce( + (acc, id) => ({ + ...acc, + [`entity.identity.${id.field}`]: { + terms: { field: id.field, missing_bucket: id.optional }, + }, + }), + {} + ), }, aggs: { ...generateLatestMetricAggregations(definition), ...generateLatestMetadataAggregations(definition), - ...generateIdentityAggregations(definition), 'entity.lastSeenTimestamp': { max: { - field: 'entity.lastSeenTimestamp', - }, - }, - 'entity.firstSeenTimestamp': { - min: { - field: '@timestamp', + field: definition.latest.timestampField, }, }, }, }, }; -} +}; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts index 7746be66f5033..12535d313143b 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts @@ -7,134 +7,22 @@ import { entityDefinitionSchema } from '@kbn/entities-schema'; import { rawEntityDefinition } from '../helpers/fixtures/entity_definition'; -import { - generateHistoryMetadataAggregations, - generateLatestMetadataAggregations, -} from './generate_metadata_aggregations'; +import { generateLatestMetadataAggregations } from './generate_metadata_aggregations'; describe('Generate Metadata Aggregations for history and latest', () => { - describe('generateHistoryMetadataAggregations()', () => { - it('should generate metadata aggregations for string format', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: ['host.name'], - }); - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.host.name': { - terms: { - field: 'host.name', - size: 1000, - }, - }, - }); - }); - - it('should generate metadata aggregations for object format with only source', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: [{ source: 'host.name' }], - }); - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.host.name': { - terms: { - field: 'host.name', - size: 1000, - }, - }, - }); - }); - - it('should generate metadata aggregations for object format with source and aggregation', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], - }); - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.host.name': { - terms: { - field: 'host.name', - size: 10, - }, - }, - }); - }); - - it('should generate metadata aggregations for object format with source, aggregation, and destination', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: [ - { - source: 'host.name', - aggregation: { type: 'terms', limit: 20 }, - destination: 'hostName', - }, - ], - }); - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.hostName': { - terms: { - field: 'host.name', - size: 20, - }, - }, - }); - }); - - it('should generate metadata aggregations for terms and top_value', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: [ - { - source: 'host.name', - aggregation: { type: 'terms', limit: 10 }, - destination: 'hostName', - }, - { - source: 'agent.name', - aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, - destination: 'agentName', - }, - ], - }); - - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.hostName': { - terms: { - field: 'host.name', - size: 10, - }, - }, - 'entity.metadata.agentName': { - filter: { - exists: { - field: 'agent.name', - }, - }, - aggs: { - top_value: { - top_metrics: { - metrics: { field: 'agent.name' }, - sort: { '@timestamp': 'desc' }, - }, - }, - }, - }, - }); - }); - }); - describe('generateLatestMetadataAggregations()', () => { it('should generate metadata aggregations for string format', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, metadata: ['host.name'], }); + expect(generateLatestMetadataAggregations(definition)).toEqual({ 'entity.metadata.host.name': { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, @@ -142,7 +30,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { data: { terms: { field: 'host.name', - size: 1000, + size: 10, }, }, }, @@ -160,7 +48,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, @@ -168,7 +56,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { data: { terms: { field: 'host.name', - size: 1000, + size: 10, }, }, }, @@ -179,14 +67,16 @@ describe('Generate Metadata Aggregations for history and latest', () => { it('should generate metadata aggregations for object format with source and aggregation', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], + metadata: [ + { source: 'host.name', aggregation: { type: 'terms', limit: 10, lookbackPeriod: '1h' } }, + ], }); expect(generateLatestMetadataAggregations(definition)).toEqual({ 'entity.metadata.host.name': { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-1h', }, }, }, @@ -218,14 +108,14 @@ describe('Generate Metadata Aggregations for history and latest', () => { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, aggs: { data: { terms: { - field: 'hostName', + field: 'host.name', size: 10, }, }, @@ -255,14 +145,14 @@ describe('Generate Metadata Aggregations for history and latest', () => { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, aggs: { data: { terms: { - field: 'hostName', + field: 'host.name', size: 10, }, }, @@ -275,13 +165,13 @@ describe('Generate Metadata Aggregations for history and latest', () => { { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, { exists: { - field: 'agentName', + field: 'agent.name', }, }, ], @@ -291,7 +181,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { top_value: { top_metrics: { metrics: { - field: 'agentName', + field: 'agent.name', }, sort: { '@timestamp': 'desc', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 0fc4464672219..796d1e25b55ec 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -6,70 +6,28 @@ */ import { EntityDefinition } from '@kbn/entities-schema'; -import { calculateOffset } from '../helpers/calculate_offset'; - -export function generateHistoryMetadataAggregations(definition: EntityDefinition) { - if (!definition.metadata) { - return {}; - } - return definition.metadata.reduce((aggs, metadata) => { - let agg; - if (metadata.aggregation.type === 'terms') { - agg = { - terms: { - field: metadata.source, - size: metadata.aggregation.limit, - }, - }; - } else if (metadata.aggregation.type === 'top_value') { - agg = { - filter: { - exists: { - field: metadata.source, - }, - }, - aggs: { - top_value: { - top_metrics: { - metrics: { - field: metadata.source, - }, - sort: metadata.aggregation.sort, - }, - }, - }, - }; - } - - return { - ...aggs, - [`entity.metadata.${metadata.destination}`]: agg, - }; - }, {}); -} export function generateLatestMetadataAggregations(definition: EntityDefinition) { if (!definition.metadata) { return {}; } - const offsetInSeconds = `${calculateOffset(definition)}s`; - return definition.metadata.reduce((aggs, metadata) => { + const lookbackPeriod = metadata.aggregation.lookbackPeriod || definition.latest.lookbackPeriod; let agg; if (metadata.aggregation.type === 'terms') { agg = { filter: { range: { '@timestamp': { - gte: `now-${offsetInSeconds}`, + gte: `now-${lookbackPeriod}`, }, }, }, aggs: { data: { terms: { - field: metadata.destination, + field: metadata.source, size: metadata.aggregation.limit, }, }, @@ -83,13 +41,13 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) { range: { '@timestamp': { - gte: `now-${metadata.aggregation.lookbackPeriod ?? offsetInSeconds}`, + gte: `now-${lookbackPeriod}`, }, }, }, { exists: { - field: metadata.destination, + field: metadata.source, }, }, ], @@ -99,7 +57,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) top_value: { top_metrics: { metrics: { - field: metadata.destination, + field: metadata.source, }, sort: metadata.aggregation.sort, }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts index bd1af365116cb..d42dd69b37eff 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts @@ -104,41 +104,15 @@ function buildMetricEquation(keyMetric: KeyMetric) { }; } -export function generateHistoryMetricAggregations(definition: EntityDefinition) { - if (!definition.metrics) { - return {}; - } - return definition.metrics.reduce((aggs, keyMetric) => { - return { - ...aggs, - ...buildMetricAggregations(keyMetric, definition.history.timestampField), - [`entity.metrics.${keyMetric.name}`]: buildMetricEquation(keyMetric), - }; - }, {}); -} - export function generateLatestMetricAggregations(definition: EntityDefinition) { if (!definition.metrics) { return {}; } - return definition.metrics.reduce((aggs, keyMetric) => { return { ...aggs, - [`_${keyMetric.name}`]: { - top_metrics: { - metrics: [{ field: `entity.metrics.${keyMetric.name}` }], - sort: [{ '@timestamp': 'desc' }], - }, - }, - [`entity.metrics.${keyMetric.name}`]: { - bucket_script: { - buckets_path: { - value: `_${keyMetric.name}[entity.metrics.${keyMetric.name}]`, - }, - script: 'params.value', - }, - }, + ...buildMetricAggregations(keyMetric, definition.latest.timestampField), + [`entity.metrics.${keyMetric.name}`]: buildMetricEquation(keyMetric), }; }, {}); } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts index c16b7f126dded..c703124bdf082 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts @@ -7,26 +7,14 @@ import { EntityDefinition } from '@kbn/entities-schema'; import { EntityDefinitionIdInvalid } from '../errors/entity_definition_id_invalid'; -import { - generateHistoryBackfillTransformId, - generateHistoryTransformId, - generateLatestTransformId, -} from '../helpers/generate_component_id'; +import { generateLatestTransformId } from '../helpers/generate_component_id'; const TRANSFORM_ID_MAX_LENGTH = 64; export function validateDefinitionCanCreateValidTransformIds(definition: EntityDefinition) { - const historyTransformId = generateHistoryTransformId(definition); const latestTransformId = generateLatestTransformId(definition); - const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); - const spareChars = - TRANSFORM_ID_MAX_LENGTH - - Math.max( - historyTransformId.length, - latestTransformId.length, - historyBackfillTransformId.length - ); + const spareChars = TRANSFORM_ID_MAX_LENGTH - latestTransformId.length; if (spareChars < 0) { throw new EntityDefinitionIdInvalid( diff --git a/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts index 8bc8efa3870aa..d0e0410b6e422 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts @@ -11,14 +11,10 @@ import { EntityDefinition } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; import { deleteEntityDefinition } from './delete_entity_definition'; import { deleteIndices } from './delete_index'; -import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline'; +import { deleteIngestPipelines } from './delete_ingest_pipeline'; import { findEntityDefinitions } from './find_entity_definition'; -import { - generateHistoryIndexTemplateId, - generateLatestIndexTemplateId, -} from './helpers/generate_component_id'; -import { deleteTemplate } from '../manage_index_templates'; +import { deleteTemplates } from '../manage_index_templates'; import { stopTransforms } from './stop_transforms'; @@ -40,19 +36,13 @@ export async function uninstallEntityDefinition({ await stopTransforms(esClient, definition, logger); await deleteTransforms(esClient, definition, logger); - await Promise.all([ - deleteHistoryIngestPipeline(esClient, definition, logger), - deleteLatestIngestPipeline(esClient, definition, logger), - ]); + await deleteIngestPipelines(esClient, definition, logger); if (deleteData) { await deleteIndices(esClient, definition, logger); } - await Promise.all([ - deleteTemplate({ esClient, logger, name: generateHistoryIndexTemplateId(definition) }), - deleteTemplate({ esClient, logger, name: generateLatestIndexTemplateId(definition) }), - ]); + await deleteTemplates(esClient, definition, logger); await deleteEntityDefinition(soClient, definition); } diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts index ee6b59b0ae0ea..710872c04eda0 100644 --- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts +++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts @@ -41,7 +41,7 @@ export class EntityClient { }); if (!installOnly) { - await startTransforms(this.options.esClient, definition, this.options.logger); + await startTransforms(this.options.esClient, installedDefinition, this.options.logger); } return installedDefinition; diff --git a/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts b/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts index b0789b6cf2769..ffa58cd9c0145 100644 --- a/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts +++ b/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { EntityDefinition } from '@kbn/entities-schema'; import { ClusterPutComponentTemplateRequest, IndicesPutIndexTemplateRequest, @@ -15,6 +16,7 @@ import { entitiesLatestBaseComponentTemplateConfig } from '../templates/componen import { entitiesEntityComponentTemplateConfig } from '../templates/components/entity'; import { entitiesEventComponentTemplateConfig } from '../templates/components/event'; import { retryTransientEsErrors } from './entities/helpers/retry'; +import { generateEntitiesLatestIndexTemplateConfig } from './entities/templates/entities_latest_template'; interface TemplateManagementOptions { esClient: ElasticsearchClient; @@ -67,14 +69,27 @@ interface DeleteTemplateOptions { export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { try { - await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); + const result = await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { + logger, + }); logger.debug(() => `Installed entity manager index template: ${JSON.stringify(template)}`); + return result; } catch (error: any) { logger.error(`Error updating entity manager index template: ${error.message}`); throw error; } } +export async function createAndInstallTemplates( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +): Promise> { + const template = generateEntitiesLatestIndexTemplateConfig(definition); + await upsertTemplate({ esClient, template, logger }); + return [{ type: 'template', id: template.name }]; +} + export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { try { await retryTransientEsErrors( @@ -87,6 +102,28 @@ export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateO } } +export async function deleteTemplates( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'template') + .map(({ id }) => + retryTransientEsErrors( + () => esClient.indices.deleteIndexTemplate({ name: id }, { ignore: [404] }), + { logger } + ) + ) + ); + } catch (error: any) { + logger.error(`Error deleting entity manager index template: ${error.message}`); + throw error; + } +} + export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { try { await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts index bde68eb85ba9f..9c1c4f403636b 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts @@ -51,8 +51,8 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ }), handler: async ({ context, response, params, logger, server }) => { try { - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const canDisable = await canDisableEntityDiscovery(esClient); + const esClientAsCurrentUser = (await context.core).elasticsearch.client.asCurrentUser; + const canDisable = await canDisableEntityDiscovery(esClientAsCurrentUser); if (!canDisable) { return response.forbidden({ body: { @@ -62,6 +62,7 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ }); } + const esClient = (await context.core).elasticsearch.client.asSecondaryAuthUser; const soClient = (await context.core).savedObjects.getClient({ includedHiddenTypes: [EntityDiscoveryApiKeyType.name], }); diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts b/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts index 9814840d20a0b..1002c1e716df2 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts @@ -80,8 +80,10 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({ }); } - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const canEnable = await canEnableEntityDiscovery(esClient); + const core = await context.core; + + const esClientAsCurrentUser = core.elasticsearch.client.asCurrentUser; + const canEnable = await canEnableEntityDiscovery(esClientAsCurrentUser); if (!canEnable) { return response.forbidden({ body: { @@ -91,7 +93,7 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({ }); } - const soClient = (await context.core).savedObjects.getClient({ + const soClient = core.savedObjects.getClient({ includedHiddenTypes: [EntityDiscoveryApiKeyType.name], }); const existingApiKey = await readEntityDiscoveryAPIKey(server); @@ -117,6 +119,7 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({ await saveEntityDiscoveryAPIKey(soClient, apiKey); + const esClient = core.elasticsearch.client.asSecondaryAuthUser; const installedDefinitions = await installBuiltInEntityDefinitions({ esClient, soClient, diff --git a/x-pack/plugins/entity_manager/server/routes/entities/reset.ts b/x-pack/plugins/entity_manager/server/routes/entities/reset.ts index a59c38b3acf7c..0b6942e335e51 100644 --- a/x-pack/plugins/entity_manager/server/routes/entities/reset.ts +++ b/x-pack/plugins/entity_manager/server/routes/entities/reset.ts @@ -12,25 +12,13 @@ import { EntitySecurityException } from '../../lib/entities/errors/entity_securi import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; -import { - deleteHistoryIngestPipeline, - deleteLatestIngestPipeline, -} from '../../lib/entities/delete_ingest_pipeline'; +import { deleteIngestPipelines } from '../../lib/entities/delete_ingest_pipeline'; import { deleteIndices } from '../../lib/entities/delete_index'; -import { - createAndInstallHistoryIngestPipeline, - createAndInstallLatestIngestPipeline, -} from '../../lib/entities/create_and_install_ingest_pipeline'; -import { - createAndInstallHistoryBackfillTransform, - createAndInstallHistoryTransform, - createAndInstallLatestTransform, -} from '../../lib/entities/create_and_install_transform'; +import { createAndInstallIngestPipelines } from '../../lib/entities/create_and_install_ingest_pipeline'; +import { createAndInstallTransforms } from '../../lib/entities/create_and_install_transform'; import { startTransforms } from '../../lib/entities/start_transforms'; import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; -import { isBackfillEnabled } from '../../lib/entities/helpers/is_backfill_enabled'; - import { createEntityManagerServerRoute } from '../create_entity_manager_server_route'; import { deleteTransforms } from '../../lib/entities/delete_transforms'; import { stopTransforms } from '../../lib/entities/stop_transforms'; @@ -51,18 +39,12 @@ export const resetEntityDefinitionRoute = createEntityManagerServerRoute({ await stopTransforms(esClient, definition, logger); await deleteTransforms(esClient, definition, logger); - await deleteHistoryIngestPipeline(esClient, definition, logger); - await deleteLatestIngestPipeline(esClient, definition, logger); + await deleteIngestPipelines(esClient, definition, logger); await deleteIndices(esClient, definition, logger); // Recreate everything - await createAndInstallHistoryIngestPipeline(esClient, definition, logger); - await createAndInstallLatestIngestPipeline(esClient, definition, logger); - await createAndInstallHistoryTransform(esClient, definition, logger); - if (isBackfillEnabled(definition)) { - await createAndInstallHistoryBackfillTransform(esClient, definition, logger); - } - await createAndInstallLatestTransform(esClient, definition, logger); + await createAndInstallIngestPipelines(esClient, definition, logger); + await createAndInstallTransforms(esClient, definition, logger); await startTransforms(esClient, definition, logger); return response.ok({ body: { acknowledged: true } }); diff --git a/x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts b/x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts index fdf2510e8627e..bdea2b71e4141 100644 --- a/x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts @@ -5,11 +5,36 @@ * 2.0. */ +import { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server'; import { SavedObject, SavedObjectsType } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; +import { + generateHistoryIndexTemplateId, + generateHistoryIngestPipelineId, + generateHistoryTransformId, + generateLatestIndexTemplateId, + generateLatestIngestPipelineId, + generateLatestTransformId, +} from '../lib/entities/helpers/generate_component_id'; export const SO_ENTITY_DEFINITION_TYPE = 'entity-definition'; +export const backfillInstalledComponents: SavedObjectModelDataBackfillFn< + EntityDefinition, + EntityDefinition +> = (savedObject) => { + const definition = savedObject.attributes; + definition.installedComponents = [ + { type: 'transform', id: generateHistoryTransformId(definition) }, + { type: 'transform', id: generateLatestTransformId(definition) }, + { type: 'ingest_pipeline', id: generateHistoryIngestPipelineId(definition) }, + { type: 'ingest_pipeline', id: generateLatestIngestPipelineId(definition) }, + { type: 'template', id: generateHistoryIndexTemplateId(definition) }, + { type: 'template', id: generateLatestIndexTemplateId(definition) }, + ]; + return savedObject; +}; + export const entityDefinition: SavedObjectsType = { name: SO_ENTITY_DEFINITION_TYPE, hidden: false, @@ -64,5 +89,13 @@ export const entityDefinition: SavedObjectsType = { }, ], }, + '3': { + changes: [ + { + type: 'data_backfill', + backfillFn: backfillInstalledComponents, + }, + ], + }, }, }; diff --git a/x-pack/plugins/entity_manager/server/templates/components/helpers.test.ts b/x-pack/plugins/entity_manager/server/templates/components/helpers.test.ts deleted file mode 100644 index 90c5e90d43f3a..0000000000000 --- a/x-pack/plugins/entity_manager/server/templates/components/helpers.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; -import { getCustomHistoryTemplateComponents, getCustomLatestTemplateComponents } from './helpers'; - -describe('helpers', () => { - it('getCustomLatestTemplateComponents should return template component in the right sort order', () => { - const result = getCustomLatestTemplateComponents({ id: 'test' } as EntityDefinition); - expect(result).toEqual([ - 'test@platform', - 'test-latest@platform', - 'test@custom', - 'test-latest@custom', - ]); - }); - - it('getCustomHistoryTemplateComponents should return template component in the right sort order', () => { - const result = getCustomHistoryTemplateComponents({ id: 'test' } as EntityDefinition); - expect(result).toEqual([ - 'test@platform', - 'test-history@platform', - 'test@custom', - 'test-history@custom', - ]); - }); -}); diff --git a/x-pack/plugins/entity_manager/server/templates/components/helpers.ts b/x-pack/plugins/entity_manager/server/templates/components/helpers.ts deleted file mode 100644 index 23cc7cccb6a13..0000000000000 --- a/x-pack/plugins/entity_manager/server/templates/components/helpers.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; -import { isBuiltinDefinition } from '../../lib/entities/helpers/is_builtin_definition'; - -export const getCustomLatestTemplateComponents = (definition: EntityDefinition) => { - if (isBuiltinDefinition(definition)) { - return []; - } - - return [ - `${definition.id}@platform`, // @platform goes before so it can be overwritten by custom - `${definition.id}-latest@platform`, - `${definition.id}@custom`, - `${definition.id}-latest@custom`, - ]; -}; - -export const getCustomHistoryTemplateComponents = (definition: EntityDefinition) => { - if (isBuiltinDefinition(definition)) { - return []; - } - - return [ - `${definition.id}@platform`, // @platform goes before so it can be overwritten by custom - `${definition.id}-history@platform`, - `${definition.id}@custom`, - `${definition.id}-history@custom`, - ]; -}; diff --git a/x-pack/plugins/entity_manager/tsconfig.json b/x-pack/plugins/entity_manager/tsconfig.json index 29c100ee4c9d2..34c57a27dd829 100644 --- a/x-pack/plugins/entity_manager/tsconfig.json +++ b/x-pack/plugins/entity_manager/tsconfig.json @@ -34,5 +34,6 @@ "@kbn/zod-helpers", "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", + "@kbn/core-saved-objects-server", ] } diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts index 8f3a0abb62b67..00151f2029d21 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts @@ -61,7 +61,6 @@ export async function getEntitiesWithSource({ identityFields: entity?.entity.identityFields, id: entity?.entity.id, definitionId: entity?.entity.definitionId, - firstSeenTimestamp: entity?.entity.firstSeenTimestamp, lastSeenTimestamp: entity?.entity.lastSeenTimestamp, displayName: entity?.entity.displayName, metrics: entity?.entity.metrics, diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts index a72e00bf7aceb..09dea151a050a 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts @@ -27,9 +27,8 @@ export const buildHostEntityDefinition = (space: string): EntityDefinition => 'host.type', 'host.architecture', ], - history: { + latest: { timestampField: '@timestamp', - interval: '1m', }, version: '1.0.0', managed: true, @@ -44,9 +43,8 @@ export const buildUserEntityDefinition = (space: string): EntityDefinition => identityFields: ['user.name'], displayNameTemplate: '{{user.name}}', metadata: ['user.email', 'user.full_name', 'user.hash', 'user.id', 'user.name', 'user.roles'], - history: { + latest: { timestampField: '@timestamp', - interval: '1m', }, version: '1.0.0', managed: true, diff --git a/x-pack/test/api_integration/apis/entity_manager/definitions.ts b/x-pack/test/api_integration/apis/entity_manager/definitions.ts index 466b5e0232bf0..b51a26ad7b5ad 100644 --- a/x-pack/test/api_integration/apis/entity_manager/definitions.ts +++ b/x-pack/test/api_integration/apis/entity_manager/definitions.ts @@ -8,10 +8,7 @@ import semver from 'semver'; import expect from '@kbn/expect'; import { entityLatestSchema } from '@kbn/entities-schema'; -import { - entityDefinition as mockDefinition, - entityDefinitionWithBackfill as mockBackfillDefinition, -} from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures'; +import { entityDefinition as mockDefinition } from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures'; import { PartialConfig, cleanup, generate } from '@kbn/data-forge'; import { generateLatestIndexName } from '@kbn/entityManager-plugin/server/lib/entities/helpers/generate_component_id'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -33,8 +30,9 @@ export default function ({ getService }: FtrProviderContext) { describe('Entity definitions', () => { describe('definitions installations', () => { it('can install multiple definitions', async () => { + const mockDefinitionDup = { ...mockDefinition, id: 'mock_definition_dup' }; await installDefinition(supertest, { definition: mockDefinition }); - await installDefinition(supertest, { definition: mockBackfillDefinition }); + await installDefinition(supertest, { definition: mockDefinitionDup }); const { definitions } = await getInstalledDefinitions(supertest); expect(definitions.length).to.eql(2); @@ -49,7 +47,7 @@ export default function ({ getService }: FtrProviderContext) { expect( definitions.some( (definition) => - definition.id === mockBackfillDefinition.id && + definition.id === mockDefinitionDup.id && definition.state.installed === true && definition.state.running === true ) @@ -57,7 +55,7 @@ export default function ({ getService }: FtrProviderContext) { await Promise.all([ uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true }), - uninstallDefinition(supertest, { id: mockBackfillDefinition.id, deleteData: true }), + uninstallDefinition(supertest, { id: mockDefinitionDup.id, deleteData: true }), ]); }); @@ -89,7 +87,7 @@ export default function ({ getService }: FtrProviderContext) { id: mockDefinition.id, update: { version: incVersion!, - history: { + latest: { timestampField: '@updatedTimestampField', }, }, @@ -99,7 +97,7 @@ export default function ({ getService }: FtrProviderContext) { definitions: [updatedDefinition], } = await getInstalledDefinitions(supertest); expect(updatedDefinition.version).to.eql(incVersion); - expect(updatedDefinition.history.timestampField).to.eql('@updatedTimestampField'); + expect(updatedDefinition.latest.timestampField).to.eql('@updatedTimestampField'); await uninstallDefinition(supertest, { id: mockDefinition.id }); }); @@ -114,7 +112,7 @@ export default function ({ getService }: FtrProviderContext) { id: mockDefinition.id, update: { version: '1.0.0', - history: { + latest: { timestampField: '@updatedTimestampField', }, }, diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index 99d84fbc5427b..4fb2360a049cf 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -27,10 +27,7 @@ export default ({ getService }: FtrProviderContext) => { it('should have installed the expected user resources', async () => { await utils.initEntityEngineForEntityType('user'); - const expectedTransforms = [ - 'entities-v1-history-ea_default_user_entity_store', - 'entities-v1-latest-ea_default_user_entity_store', - ]; + const expectedTransforms = ['entities-v1-latest-ea_default_user_entity_store']; await utils.expectTransformsExist(expectedTransforms); }); @@ -38,10 +35,7 @@ export default ({ getService }: FtrProviderContext) => { it('should have installed the expected host resources', async () => { await utils.initEntityEngineForEntityType('host'); - const expectedTransforms = [ - 'entities-v1-history-ea_default_host_entity_store', - 'entities-v1-latest-ea_default_host_entity_store', - ]; + const expectedTransforms = ['entities-v1-latest-ea_default_host_entity_store']; await utils.expectTransformsExist(expectedTransforms); }); @@ -173,7 +167,6 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - await utils.expectTransformNotFound('entities-v1-history-ea_host_entity_store'); await utils.expectTransformNotFound('entities-v1-latest-ea_host_entity_store'); }); @@ -187,7 +180,6 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - await utils.expectTransformNotFound('entities-v1-history-ea_user_entity_store'); await utils.expectTransformNotFound('entities-v1-latest-ea_user_entity_store'); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index 112c8b8b21511..e3ef29d937183 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -38,10 +38,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { it('should have installed the expected user resources', async () => { await utils.initEntityEngineForEntityType('user'); - const expectedTransforms = [ - `entities-v1-history-ea_${namespace}_user_entity_store`, - `entities-v1-latest-ea_${namespace}_user_entity_store`, - ]; + const expectedTransforms = [`entities-v1-latest-ea_${namespace}_user_entity_store`]; await utils.expectTransformsExist(expectedTransforms); }); @@ -49,10 +46,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { it('should have installed the expected host resources', async () => { await utils.initEntityEngineForEntityType('host'); - const expectedTransforms = [ - `entities-v1-history-ea_${namespace}_host_entity_store`, - `entities-v1-latest-ea_${namespace}_host_entity_store`, - ]; + const expectedTransforms = [`entities-v1-latest-ea_${namespace}_host_entity_store`]; await utils.expectTransformsExist(expectedTransforms); });