[EEM] Calculate the latest metadata lookback based on the calculated …

…history delay (#191324)

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/calculate_offset.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition } from '@kbn/entities-schema';
+import moment from 'moment';
+import {
+  ENTITY_DEFAULT_HISTORY_FREQUENCY,
+  ENTITY_DEFAULT_HISTORY_SYNC_DELAY,
+} from '../../../../common/constants_entities';
+
+const durationToSeconds = (dateMath: string) => {
+  const parts = dateMath.match(/(\d+)([m|s|h|d])/);
+  if (!parts) {
+    throw new Error(`Invalid date math supplied: ${dateMath}`);
+  }
+  const value = parseInt(parts[1], 10);
+  const unit = parts[2] as 'm' | 's' | 'h' | 'd';
+  return moment.duration(value, unit).asSeconds();
+};
+
+export function calculateOffset(definition: EntityDefinition) {
+  const syncDelay = durationToSeconds(
+    definition.history.settings.syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY
+  );
+  const frequency =
+    durationToSeconds(definition.history.settings.frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY) *
+    2;
+
+  return syncDelay + frequency;
+}

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts

@@ -15,6 +15,11 @@ export const rawEntityDefinition = {
   history: {
     timestampField: '@timestamp',
     interval: '1m',
+    settings: {
+      lookbackPeriod: '10m',
+      frequency: '2m',
+      syncDelay: '2m',
+    },
   },
   identityFields: ['log.logger', { field: 'event.category', optional: true }],
   displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}',

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts

@@ -86,7 +86,7 @@ describe('Generate Metadata Aggregations for history and latest', () => {
           filter: {
             range: {
               '@timestamp': {
-                gte: 'now-1m',
+                gte: 'now-360s',
               },
             },
           },
@@ -112,7 +112,7 @@ describe('Generate Metadata Aggregations for history and latest', () => {
           filter: {
             range: {
               '@timestamp': {
-                gte: 'now-1m',
+                gte: 'now-360s',
               },
             },
           },
@@ -138,7 +138,7 @@ describe('Generate Metadata Aggregations for history and latest', () => {
           filter: {
             range: {
               '@timestamp': {
-                gte: 'now-1m',
+                gte: 'now-360s',
               },
             },
           },
@@ -164,7 +164,7 @@ describe('Generate Metadata Aggregations for history and latest', () => {
           filter: {
             range: {
               '@timestamp': {
-                gte: 'now-1m',
+                gte: 'now-360s',
               },
             },
           },

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts

@@ -7,6 +7,7 @@
 
 import { EntityDefinition } from '@kbn/entities-schema';
 import { ENTITY_DEFAULT_METADATA_LIMIT } from '../../../../common/constants_entities';
+import { calculateOffset } from '../helpers/calculate_offset';
 
 export function generateHistoryMetadataAggregations(definition: EntityDefinition) {
   if (!definition.metadata) {
@@ -31,14 +32,16 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition)
     return {};
   }
 
+  const offsetInSeconds = calculateOffset(definition);
+
   return definition.metadata.reduce(
     (aggs, metadata) => ({
       ...aggs,
       [`entity.metadata.${metadata.destination}`]: {
         filter: {
           range: {
             '@timestamp': {
-              gte: `now-${definition.history.interval}`,
+              gte: `now-${offsetInSeconds}s`,
             },
           },
         },