[eem] _search sort_by and display_name (#202361)

- allow optional `entity_source.display_name` setting. `entity.display_name` will always be set on the entities, falling back to `entity.id` if provided field is not set - allow `sort_by` parameter to `_search` API - removed the `metadata.` prefix in the query aggregation. metadata will now be set at the root of the document (eg for metadata `host.name` entity = `{ entity.id: 'foo', host.name: 'bar' }` - timestamp_field is now optional --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Dec 2, 2024 · 697af57 · 697af57
1 parent 54370b2
commit 697af57
Show file tree

Hide file tree

Showing 8 changed files with 309 additions and 119 deletions.
diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
@@ -25,8 +25,9 @@ export interface MetadataRecord {
 
 export interface EntityV2 {
   'entity.id': string;
-  'entity.last_seen_timestamp': string;
   'entity.type': string;
+  'entity.display_name': string;
+  'entity.last_seen_timestamp'?: string;
   [metadata: string]: any;
 }
 

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { without } from 'lodash';
 import { EntityV2, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
@@ -23,10 +24,27 @@ import { stopTransforms } from './entities/stop_transforms';
 import { deleteIndices } from './entities/delete_index';
 import { EntityDefinitionWithState } from './entities/types';
 import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
-import { EntitySource, getEntityInstancesQuery } from './queries';
+import { EntitySource, SortBy, getEntityInstancesQuery } from './queries';
 import { mergeEntitiesList, runESQLQuery } from './queries/utils';
 import { UnknownEntityType } from './entities/errors/unknown_entity_type';
 
+interface SearchCommon {
+  start: string;
+  end: string;
+  sort?: SortBy;
+  metadataFields?: string[];
+  filters?: string[];
+  limit?: number;
+}
+
+export type SearchByType = SearchCommon & {
+  type: string;
+};
+
+export type SearchBySources = SearchCommon & {
+  sources: EntitySource[];
+};
+
 export class EntityClient {
   constructor(
     private options: {
@@ -191,17 +209,11 @@ export class EntityClient {
     type,
     start,
     end,
+    sort,
     metadataFields = [],
     filters = [],
     limit = 10,
-  }: {
-    type: string;
-    start: string;
-    end: string;
-    metadataFields?: string[];
-    filters?: string[];
-    limit?: number;
-  }) {
+  }: SearchByType) {
     const sources = await this.getEntitySources({ type });
     if (sources.length === 0) {
       throw new UnknownEntityType(`No sources found for entity type [${type}]`);
@@ -213,6 +225,7 @@ export class EntityClient {
       end,
       metadataFields,
       filters,
+      sort,
       limit,
     });
   }
@@ -221,21 +234,22 @@ export class EntityClient {
     sources,
     start,
     end,
+    sort,
     metadataFields = [],
     filters = [],
     limit = 10,
-  }: {
-    sources: EntitySource[];
-    start: string;
-    end: string;
-    metadataFields?: string[];
-    filters?: string[];
-    limit?: number;
-  }) {
+  }: SearchBySources) {
     const entities = await Promise.all(
       sources.map(async (source) => {
-        const mandatoryFields = [source.timestamp_field, ...source.identity_fields];
+        const mandatoryFields = [
+          ...source.identity_fields,
+          ...(source.timestamp_field ? [source.timestamp_field] : []),
+          ...(source.display_name ? [source.display_name] : []),
+        ];
         const metaFields = [...metadataFields, ...source.metadata_fields];
+
+        // operations on an unmapped field result in a failing query so we verify
+        // field capabilities beforehand
         const { fields } = await this.options.esClient.fieldCaps({
           index: source.index_patterns,
           fields: [...mandatoryFields, ...metaFields],
@@ -244,15 +258,25 @@ export class EntityClient {
         const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]);
         if (!sourceHasMandatoryFields) {
           // we can't build entities without id fields so we ignore the source.
-          // filters should likely behave similarly.
+          // TODO filters should likely behave similarly. we should also throw
+          const missingFields = mandatoryFields.filter((field) => !fields[field]);
           this.options.logger.info(
-            `Ignoring source for type [${source.type}] with index_patterns [${source.index_patterns}] because some mandatory fields [${mandatoryFields}] are not mapped`
+            `Ignoring source for type [${source.type}] with index_patterns [${
+              source.index_patterns
+            }] because some mandatory fields [${missingFields.join(', ')}] are not mapped`
           );
           return [];
         }
 
         // but metadata field not being available is fine
         const availableMetadataFields = metaFields.filter((field) => fields[field]);
+        if (availableMetadataFields.length < metaFields.length) {
+          this.options.logger.info(
+            `Ignoring unmapped fields [${without(metaFields, ...availableMetadataFields).join(
+              ', '
+            )}]`
+          );
+        }
 
         const query = getEntityInstancesQuery({
           source: {
@@ -262,6 +286,7 @@ export class EntityClient {
           },
           start,
           end,
+          sort,
           limit,
         });
         this.options.logger.debug(`Entity query: ${query}`);
@@ -271,14 +296,10 @@ export class EntityClient {
           esClient: this.options.esClient,
         });
 
-        return rawEntities.map((entity) => {
-          entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':');
-          entity['entity.type'] = source.type;
-          return entity;
-        });
+        return rawEntities;
       })
     ).then((results) => results.flat());
 
-    return mergeEntitiesList(entities).slice(0, limit);
+    return mergeEntitiesList(sources, entities).slice(0, limit);
   }
 }
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
@@ -18,19 +18,21 @@ describe('getEntityInstancesQuery', () => {
           metadata_fields: ['host.name'],
           filters: [],
           timestamp_field: 'custom_timestamp_field',
+          display_name: 'service.id',
         },
         limit: 5,
         start: '2024-11-20T19:00:00.000Z',
         end: '2024-11-20T20:00:00.000Z',
+        sort: { field: 'entity.id', direction: 'DESC' },
       });
 
       expect(query).toEqual(
-        'FROM logs-*,metrics-*|' +
-          'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z"|' +
-          'WHERE custom_timestamp_field <= "2024-11-20T20:00:00.000Z"|' +
-          'WHERE service.name IS NOT NULL|' +
-          'STATS entity.last_seen_timestamp=MAX(custom_timestamp_field),metadata.host.name=VALUES(host.name) BY service.name|' +
-          'SORT entity.last_seen_timestamp DESC|' +
+        'FROM logs-*, metrics-* | ' +
+          'WHERE service.name IS NOT NULL | ' +
+          'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z" AND custom_timestamp_field <= "2024-11-20T20:00:00.000Z" | ' +
+          'STATS host.name = VALUES(host.name), entity.last_seen_timestamp = MAX(custom_timestamp_field), service.id = MAX(service.id) BY service.name | ' +
+          'EVAL entity.type = "service", entity.id = service.name, entity.display_name = COALESCE(service.id, entity.id) | ' +
+          'SORT entity.id DESC | ' +
           'LIMIT 5'
       );
     });

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -9,29 +9,35 @@ import { z } from '@kbn/zod';
 
 export const entitySourceSchema = z.object({
   type: z.string(),
-  timestamp_field: z.optional(z.string()).default('@timestamp'),
+  timestamp_field: z.optional(z.string()),
   index_patterns: z.array(z.string()),
   identity_fields: z.array(z.string()),
   metadata_fields: z.array(z.string()),
   filters: z.array(z.string()),
+  display_name: z.optional(z.string()),
 });
 
+export interface SortBy {
+  field: string;
+  direction: 'ASC' | 'DESC';
+}
+
 export type EntitySource = z.infer<typeof entitySourceSchema>;
 
 const sourceCommand = ({ source }: { source: EntitySource }) => {
-  let query = `FROM ${source.index_patterns}`;
+  let query = `FROM ${source.index_patterns.join(', ')}`;
 
   const esMetadataFields = source.metadata_fields.filter((field) =>
     ['_index', '_id'].includes(field)
   );
   if (esMetadataFields.length) {
-    query += ` METADATA ${esMetadataFields.join(',')}`;
+    query += ` METADATA ${esMetadataFields.join(', ')}`;
   }
 
   return query;
 };
 
-const filterCommands = ({
+const whereCommand = ({
   source,
   start,
   end,
@@ -40,52 +46,88 @@ const filterCommands = ({
   start: string;
   end: string;
 }) => {
-  const commands = [
-    `WHERE ${source.timestamp_field} >= "${start}"`,
-    `WHERE ${source.timestamp_field} <= "${end}"`,
+  const filters = [
+    source.identity_fields.map((field) => `${field} IS NOT NULL`).join(' AND '),
+    ...source.filters,
   ];
 
-  source.identity_fields.forEach((field) => {
-    commands.push(`WHERE ${field} IS NOT NULL`);
-  });
-
-  source.filters.forEach((filter) => {
-    commands.push(`WHERE ${filter}`);
-  });
+  if (source.timestamp_field) {
+    filters.push(
+      `${source.timestamp_field} >= "${start}" AND ${source.timestamp_field} <= "${end}"`
+    );
+  }
 
-  return commands;
+  return filters.map((filter) => `WHERE ${filter}`).join(' | ');
 };
 
 const statsCommand = ({ source }: { source: EntitySource }) => {
-  const aggs = [
-    // default 'last_seen' attribute
-    `entity.last_seen_timestamp=MAX(${source.timestamp_field})`,
-    ...source.metadata_fields
-      .filter((field) => !source.identity_fields.some((idField) => idField === field))
-      .map((field) => `metadata.${field}=VALUES(${field})`),
-  ];
+  const aggs = source.metadata_fields
+    .filter((field) => !source.identity_fields.some((idField) => idField === field))
+    .map((field) => `${field} = VALUES(${field})`);
+
+  if (source.timestamp_field) {
+    aggs.push(`entity.last_seen_timestamp = MAX(${source.timestamp_field})`);
+  }
+
+  if (source.display_name) {
+    // ideally we want the latest value but there's no command yet
+    // so we use MAX for now
+    aggs.push(`${source.display_name} = MAX(${source.display_name})`);
+  }
+
+  return `STATS ${aggs.join(', ')} BY ${source.identity_fields.join(', ')}`;
+};
+
+const evalCommand = ({ source }: { source: EntitySource }) => {
+  const id =
+    source.identity_fields.length === 1
+      ? source.identity_fields[0]
+      : `CONCAT(${source.identity_fields.join(', ":", ')})`;
+
+  const displayName = source.display_name
+    ? `COALESCE(${source.display_name}, entity.id)`
+    : 'entity.id';
+
+  return `EVAL ${[
+    `entity.type = "${source.type}"`,
+    `entity.id = ${id}`,
+    `entity.display_name = ${displayName}`,
+  ].join(', ')}`;
+};
+
+const sortCommand = ({ source, sort }: { source: EntitySource; sort?: SortBy }) => {
+  if (sort) {
+    return `SORT ${sort.field} ${sort.direction}`;
+  }
+
+  if (source.timestamp_field) {
+    return `SORT entity.last_seen_timestamp DESC`;
+  }
 
-  return `STATS ${aggs.join(',')} BY ${source.identity_fields.join(',')}`;
+  return `SORT entity.id ASC`;
 };
 
 export function getEntityInstancesQuery({
   source,
   limit,
   start,
   end,
+  sort,
 }: {
   source: EntitySource;
   limit: number;
   start: string;
   end: string;
+  sort?: SortBy;
 }): string {
   const commands = [
     sourceCommand({ source }),
-    ...filterCommands({ source, start, end }),
+    whereCommand({ source, start, end }),
     statsCommand({ source }),
-    `SORT entity.last_seen_timestamp DESC`,
+    evalCommand({ source }),
+    sortCommand({ source, sort }),
     `LIMIT ${limit}`,
   ];
 
-  return commands.join('|');
+  return commands.join(' | ');
 }