[eem] remove history transforms (#193999)

### Summary Remove history and backfill transforms, leaving latest transform in place. Notable changes to latest transform: - it does not read from history output anymore but source indices defined on the definition - it defines a `latest.lookbackPeriod` to limit the amount of data ingested, which defaults to 24h - each metadata aggregation now accepts a `metadata.aggregation.lookbackPeriod` which defaults to the `latest.lookbackPeriod` - `entity.firstSeenTimestamp` is removed. this should be temporary until we have a solution for elastic/elastic-entity-model#174 - latest metrics used to get the latest pre-computed value from history data, but is it now aggregating over the `lookbackPeriod` in the source indices (which can be filtered down with `metrics.filter`) - `latest` block on the entity definition is now mandatory --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Mark Hopkin <[email protected]> (cherry picked from commit 8f8e988)
elastic · Oct 9, 2024 · 845b946 · 845b946
1 parent a3da125
commit 845b946
Show file tree

Hide file tree

Showing 59 changed files with 712 additions and 2,395 deletions.
diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -91,7 +91,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e",
         "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
-        "entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9",
+        "entity-definition": "e3811fd5fbb878d170067c0d6897a2e63010af36",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
         "entity-engine-status": "0738aa1a06d3361911740f8f166071ea43a00927",
         "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",

diff --git a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap
diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { durationSchema, metadataSchema, semVerSchema, historySettingsSchema } from './common';
+import { durationSchema, metadataSchema, semVerSchema } from './common';
 
 describe('schemas', () => {
   describe('metadataSchema', () => {
@@ -66,7 +66,7 @@ describe('schemas', () => {
       expect(result.data).toEqual({
         source: 'host.name',
         destination: 'hostName',
-        aggregation: { type: 'terms', limit: 1000 },
+        aggregation: { type: 'terms', limit: 10, lookbackPeriod: undefined },
       });
     });
 
@@ -139,30 +139,4 @@ describe('schemas', () => {
       expect(result).toMatchSnapshot();
     });
   });
-
-  describe('historySettingsSchema', () => {
-    it('should return default values when not defined', () => {
-      let result = historySettingsSchema.safeParse(undefined);
-      expect(result.success).toBeTruthy();
-      expect(result.data).toEqual({ lookbackPeriod: '1h' });
-
-      result = historySettingsSchema.safeParse({ syncDelay: '1m' });
-      expect(result.success).toBeTruthy();
-      expect(result.data).toEqual({ syncDelay: '1m', lookbackPeriod: '1h' });
-    });
-
-    it('should return user defined values when defined', () => {
-      const result = historySettingsSchema.safeParse({
-        lookbackPeriod: '30m',
-        syncField: 'event.ingested',
-        syncDelay: '5m',
-      });
-      expect(result.success).toBeTruthy();
-      expect(result.data).toEqual({
-        lookbackPeriod: '30m',
-        syncField: 'event.ingested',
-        syncDelay: '5m',
-      });
-    });
-  });
 });
diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts
@@ -85,7 +85,11 @@ export const keyMetricSchema = z.object({
 export type KeyMetric = z.infer<typeof keyMetricSchema>;
 
 export const metadataAggregation = z.union([
-  z.object({ type: z.literal('terms'), limit: z.number().default(1000) }),
+  z.object({
+    type: z.literal('terms'),
+    limit: z.number().default(10),
+    lookbackPeriod: z.optional(durationSchema),
+  }),
   z.object({
     type: z.literal('top_value'),
     sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])),
@@ -99,13 +103,13 @@ export const metadataSchema = z
     destination: z.optional(z.string()),
     aggregation: z
       .optional(metadataAggregation)
-      .default({ type: z.literal('terms').value, limit: 1000 }),
+      .default({ type: z.literal('terms').value, limit: 10, lookbackPeriod: undefined }),
   })
   .or(
     z.string().transform((value) => ({
       source: value,
       destination: value,
-      aggregation: { type: z.literal('terms').value, limit: 1000 },
+      aggregation: { type: z.literal('terms').value, limit: 10, lookbackPeriod: undefined },
     }))
   )
   .transform((metadata) => ({

diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
@@ -35,7 +35,6 @@ export const entityLatestSchema = z
     entity: entityBaseSchema.merge(
       z.object({
         lastSeenTimestamp: z.string(),
-        firstSeenTimestamp: z.string(),
       })
     ),
   })

diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts
@@ -14,8 +14,6 @@ import {
   durationSchema,
   identityFieldsSchema,
   semVerSchema,
-  historySettingsSchema,
-  durationSchemaWithMinimum,
 } from './common';
 
 export const entityDefinitionSchema = z.object({
@@ -32,22 +30,17 @@ export const entityDefinitionSchema = z.object({
   metrics: z.optional(z.array(keyMetricSchema)),
   staticFields: z.optional(z.record(z.string(), z.string())),
   managed: z.optional(z.boolean()).default(false),
-  history: z.object({
+  latest: z.object({
     timestampField: z.string(),
-    interval: durationSchemaWithMinimum(1),
-    settings: historySettingsSchema,
+    lookbackPeriod: z.optional(durationSchema).default('24h'),
+    settings: z.optional(
+      z.object({
+        syncField: z.optional(z.string()),
+        syncDelay: z.optional(durationSchema),
+        frequency: z.optional(durationSchema),
+      })
+    ),
   }),
-  latest: z.optional(
-    z.object({
-      settings: z.optional(
-        z.object({
-          syncField: z.optional(z.string()),
-          syncDelay: z.optional(durationSchema),
-          frequency: z.optional(durationSchema),
-        })
-      ),
-    })
-  ),
   installStatus: z.optional(
     z.union([
       z.literal('installing'),
@@ -57,6 +50,18 @@ export const entityDefinitionSchema = z.object({
     ])
   ),
   installStartedAt: z.optional(z.string()),
+  installedComponents: z.optional(
+    z.array(
+      z.object({
+        type: z.union([
+          z.literal('transform'),
+          z.literal('ingest_pipeline'),
+          z.literal('template'),
+        ]),
+        id: z.string(),
+      })
+    )
+  ),
 });
 
 export const entityDefinitionUpdateSchema = entityDefinitionSchema
@@ -69,7 +74,7 @@ export const entityDefinitionUpdateSchema = entityDefinitionSchema
   .partial()
   .merge(
     z.object({
-      history: z.optional(entityDefinitionSchema.shape.history.partial()),
+      latest: z.optional(entityDefinitionSchema.shape.latest.partial()),
       version: semVerSchema,
     })
   );

diff --git a/x-pack/plugins/entity_manager/common/constants_entities.ts b/x-pack/plugins/entity_manager/common/constants_entities.ts
@@ -33,8 +33,6 @@ export const ENTITY_LATEST_PREFIX_V1 =
   `${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_LATEST}` as const;
 
 // Transform constants
-export const ENTITY_DEFAULT_HISTORY_FREQUENCY = '1m';
-export const ENTITY_DEFAULT_HISTORY_SYNC_DELAY = '60s';
-export const ENTITY_DEFAULT_LATEST_FREQUENCY = '30s';
-export const ENTITY_DEFAULT_LATEST_SYNC_DELAY = '1s';
-export const ENTITY_DEFAULT_METADATA_LIMIT = 1000;
+export const ENTITY_DEFAULT_LATEST_FREQUENCY = '1m';
+export const ENTITY_DEFAULT_LATEST_SYNC_DELAY = '60s';
+export const ENTITY_DEFAULT_METADATA_LIMIT = 10;
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts
@@ -20,9 +20,9 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition =
     indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'],
     identityFields: ['container.id'],
     displayNameTemplate: '{{container.id}}',
-    history: {
+    latest: {
       timestampField: '@timestamp',
-      interval: '5m',
+      lookbackPeriod: '10m',
       settings: {
         frequency: '5m',
       },

diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts
@@ -19,9 +19,9 @@ export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefin
   indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'],
   identityFields: ['host.name'],
   displayNameTemplate: '{{host.name}}',
-  history: {
+  latest: {
     timestampField: '@timestamp',
-    interval: '5m',
+    lookbackPeriod: '10m',
     settings: {
       frequency: '5m',
     },

diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts
@@ -18,11 +18,10 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
     type: 'service',
     managed: true,
     indexPatterns: ['logs-*', 'filebeat*', 'traces-apm*'],
-    history: {
+    latest: {
       timestampField: '@timestamp',
-      interval: '1m',
+      lookbackPeriod: '10m',
       settings: {
-        lookbackPeriod: '10m',
         frequency: '2m',
         syncDelay: '2m',
       },

diff --git a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts
@@ -7,46 +7,15 @@
 
 import { ElasticsearchClient, Logger } from '@kbn/core/server';
 import { EntityDefinition } from '@kbn/entities-schema';
-import {
-  generateHistoryIngestPipelineId,
-  generateLatestIngestPipelineId,
-} from './helpers/generate_component_id';
+import { generateLatestIngestPipelineId } from './helpers/generate_component_id';
 import { retryTransientEsErrors } from './helpers/retry';
-import { generateHistoryProcessors } from './ingest_pipeline/generate_history_processors';
 import { generateLatestProcessors } from './ingest_pipeline/generate_latest_processors';
 
-export async function createAndInstallHistoryIngestPipeline(
+export async function createAndInstallIngestPipelines(
   esClient: ElasticsearchClient,
   definition: EntityDefinition,
   logger: Logger
-) {
-  try {
-    const historyProcessors = generateHistoryProcessors(definition);
-    const historyId = generateHistoryIngestPipelineId(definition);
-    await retryTransientEsErrors(
-      () =>
-        esClient.ingest.putPipeline({
-          id: historyId,
-          processors: historyProcessors,
-          _meta: {
-            definitionVersion: definition.version,
-            managed: definition.managed,
-          },
-        }),
-      { logger }
-    );
-  } catch (e) {
-    logger.error(
-      `Cannot create entity history ingest pipelines for [${definition.id}] entity defintion`
-    );
-    throw e;
-  }
-}
-export async function createAndInstallLatestIngestPipeline(
-  esClient: ElasticsearchClient,
-  definition: EntityDefinition,
-  logger: Logger
-) {
+): Promise<Array<{ type: 'ingest_pipeline'; id: string }>> {
   try {
     const latestProcessors = generateLatestProcessors(definition);
     const latestId = generateLatestIngestPipelineId(definition);
@@ -62,9 +31,10 @@ export async function createAndInstallLatestIngestPipeline(
         }),
       { logger }
     );
+    return [{ type: 'ingest_pipeline', id: latestId }];
   } catch (e) {
     logger.error(
-      `Cannot create entity latest ingest pipelines for [${definition.id}] entity defintion`
+      `Cannot create entity latest ingest pipelines for [${definition.id}] entity definition`
     );
     throw e;
   }

diff --git a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts
@@ -9,57 +9,20 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server';
 import { EntityDefinition } from '@kbn/entities-schema';
 import { retryTransientEsErrors } from './helpers/retry';
 import { generateLatestTransform } from './transform/generate_latest_transform';
-import {
-  generateBackfillHistoryTransform,
-  generateHistoryTransform,
-} from './transform/generate_history_transform';
 
-export async function createAndInstallHistoryTransform(
+export async function createAndInstallTransforms(
   esClient: ElasticsearchClient,
   definition: EntityDefinition,
   logger: Logger
-) {
-  try {
-    const historyTransform = generateHistoryTransform(definition);
-    await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), {
-      logger,
-    });
-  } catch (e) {
-    logger.error(`Cannot create entity history transform for [${definition.id}] entity definition`);
-    throw e;
-  }
-}
-
-export async function createAndInstallHistoryBackfillTransform(
-  esClient: ElasticsearchClient,
-  definition: EntityDefinition,
-  logger: Logger
-) {
-  try {
-    const historyTransform = generateBackfillHistoryTransform(definition);
-    await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), {
-      logger,
-    });
-  } catch (e) {
-    logger.error(
-      `Cannot create entity history backfill transform for [${definition.id}] entity definition`
-    );
-    throw e;
-  }
-}
-
-export async function createAndInstallLatestTransform(
-  esClient: ElasticsearchClient,
-  definition: EntityDefinition,
-  logger: Logger
-) {
+): Promise<Array<{ type: 'transform'; id: string }>> {
   try {
     const latestTransform = generateLatestTransform(definition);
     await retryTransientEsErrors(() => esClient.transform.putTransform(latestTransform), {
       logger,
     });
+    return [{ type: 'transform', id: latestTransform.transform_id }];
   } catch (e) {
-    logger.error(`Cannot create entity latest transform for [${definition.id}] entity definition`);
+    logger.error(`Cannot create entity history transform for [${definition.id}] entity definition`);
     throw e;
   }
 }