[8.16] Backport: [eem] rename fields to snake case #195895

packages/kbn-apm-synthtrace-client/index.ts

@@ -37,3 +37,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator }
 export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs';
 export { type AssetDocument } from './src/lib/assets';
 export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics';
+export { type EntityFields, entities } from './src/lib/entities';

packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class ContainerEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'container',
+      'entity.definition_id': 'builtin_containers_from_ecs_data',
+      'entity.identity_fields': ['container.id'],
+    });
+  }
+}
+
+export function containerEntity({
+  agentName,
+  dataStreamType,
+  containerId,
+  entityId,
+}: {
+  agentName: string[];
+  dataStreamType: EntityDataStreamType[];
+  containerId: string;
+  entityId: string;
+}) {
+  return new ContainerEntity({
+    'source_data_stream.type': dataStreamType,
+    'agent.name': agentName,
+    'container.id': containerId,
+    'entity.display_name': containerId,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class HostEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'host',
+      'entity.definition_id': 'builtin_hosts_from_ecs_data',
+      'entity.identity_fields': ['host.name'],
+    });
+  }
+}
+
+export function hostEntity({
+  agentName,
+  dataStreamType,
+  hostName,
+  entityId,
+}: {
+  agentName: string[];
+  dataStreamType: EntityDataStreamType[];
+  hostName: string;
+  entityId: string;
+}) {
+  return new HostEntity({
+    'source_data_stream.type': dataStreamType,
+    'agent.name': agentName,
+    'host.name': hostName,
+    'entity.display_name': hostName,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Fields } from '../entity';
+import { serviceEntity } from './service_entity';
+import { hostEntity } from './host_entity';
+import { containerEntity } from './container_entity';
+
+export type EntityDataStreamType = 'metrics' | 'logs' | 'traces';
+export type Schema = 'ecs' | 'semconv';
+
+export type EntityFields = Fields &
+  Partial<{
+    'agent.name': string[];
+    'source_data_stream.type': string | string[];
+    'source_data_stream.dataset': string | string[];
+    'event.ingested': string;
+    source_index: string;
+    'entity.last_seen_timestamp': string;
+    'entity.schema_version': string;
+    'entity.definition_version': string;
+    'entity.display_name': string;
+    'entity.identity_fields': string | string[];
+    'entity.id': string;
+    'entity.type': string;
+    'entity.definition_id': string;
+    [key: string]: any;
+  }>;
+
+export const entities = { serviceEntity, hostEntity, containerEntity };

packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class ServiceEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'service',
+      'entity.definition_id': 'builtin_services_from_ecs_data',
+      'entity.identity_fields': ['service.name'],
+    });
+  }
+}
+
+export function serviceEntity({
+  agentName,
+  dataStreamType,
+  serviceName,
+  environment,
+  entityId,
+}: {
+  agentName: string[];
+  serviceName: string;
+  dataStreamType: EntityDataStreamType[];
+  environment?: string;
+  entityId: string;
+}) {
+  return new ServiceEntity({
+    'service.name': serviceName,
+    'entity.display_name': serviceName,
+    'service.environment': environment,
+    'source_data_stream.type': dataStreamType,
+    'agent.name': agentName,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { EntityFields, ESDocumentWithOperation } from '@kbn/apm-synthtrace-client';
+import { pipeline, Readable, Transform } from 'stream';
+import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client';
+import { getDedotTransform } from '../shared/get_dedot_transform';
+import { getSerializeTransform } from '../shared/get_serialize_transform';
+import { Logger } from '../utils/create_logger';
+
+export type EntitiesSynthtraceEsClientOptions = Omit<SynthtraceEsClientOptions, 'pipeline'>;
+
+interface Pipeline {
+  includeSerialization?: boolean;
+}
+
+export class EntitiesSynthtraceEsClient extends SynthtraceEsClient<EntityFields> {
+  constructor(options: { client: Client; logger: Logger } & EntitiesSynthtraceEsClientOptions) {
+    super({
+      ...options,
+      pipeline: entitiesPipeline(),
+    });
+    this.indices = ['.entities.v1.latest.builtin*'];
+  }
+
+  getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
+    return entitiesPipeline({ includeSerialization });
+  }
+}
+
+function entitiesPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
+  return (base: Readable) => {
+    const serializationTransform = includeSerialization ? [getSerializeTransform()] : [];
+
+    return pipeline(
+      // @ts-expect-error Some weird stuff here with the type definition for pipeline. We have tests!
+      base,
+      ...serializationTransform,
+      lastSeenTimestampTransform(),
+      getRoutingTransform(),
+      getDedotTransform(),
+      (err: unknown) => {
+        if (err) {
+          throw err;
+        }
+      }
+    );
+  };
+}
+
+function lastSeenTimestampTransform() {
+  return new Transform({
+    objectMode: true,
+    transform(document: ESDocumentWithOperation<EntityFields>, encoding, callback) {
+      const timestamp = document['@timestamp'];
+      if (timestamp) {
+        const isoString = new Date(timestamp).toISOString();
+        document['entity.last_seen_timestamp'] = isoString;
+        document['event.ingested'] = isoString;
+        delete document['@timestamp'];
+      }
+      callback(null, document);
+    },
+  });
+}
+
+function getRoutingTransform() {
+  return new Transform({
+    objectMode: true,
+    transform(document: ESDocumentWithOperation<EntityFields>, encoding, callback) {
+      const entityType: string | undefined = document['entity.type'];
+      if (entityType === undefined) {
+        throw new Error(`entity.type was not defined: ${JSON.stringify(document)}`);
+      }
+      const entityIndexName = `${entityType}s`;
+      document._action = {
+        index: {
+          _index:
+            `.entities.v1.latest.builtin_${entityIndexName}_from_ecs_data`.toLocaleLowerCase(),
+          _id: document['entity.id'],
+        },
+      };
+
+      callback(null, document);
+    },
+  });
+}

packages/kbn-investigation-shared/src/rest_specs/entity.ts

@@ -19,13 +19,12 @@ const metricsSchema = z.object({
 
 const entitySchema = z.object({
   id: z.string(),
-  definitionId: z.string(),
-  definitionVersion: z.string(),
-  displayName: z.string(),
-  firstSeenTimestamp: z.string(),
-  lastSeenTimestamp: z.string(),
-  identityFields: z.array(z.string()),
-  schemaVersion: z.string(),
+  definition_id: z.string(),
+  definition_version: z.string(),
+  display_name: z.string(),
+  last_seen_timestamp: z.string(),
+  identity_fields: z.array(z.string()),
+  schema_version: z.string(),
   type: z.string(),
   metrics: metricsSchema,
 });

x-pack/packages/kbn-entities-schema/src/schema/entity.test.ts

@@ -9,19 +9,15 @@ import { entityLatestSchema, entityMetadataSchema } from './entity';
 
 const entity = {
   entity: {
-    lastSeenTimestamp: '2024-08-06T17:03:50.722Z',
-    schemaVersion: 'v1',
-    definitionVersion: '999.999.999',
-    displayName: 'message_processor',
-    identityFields: ['log.logger', 'event.category'],
+    last_seen_timestamp: '2024-08-06T17:03:50.722Z',
+    schema_version: 'v1',
+    definition_version: '999.999.999',
+    display_name: 'message_processor',
+    identity_fields: ['log.logger', 'event.category'],
     id: '6UHVPiduEC2qk6rMjs1Jzg==',
-    metrics: {
-      logRate: 100,
-      errorRate: 0,
-    },
     type: 'service',
-    firstSeenTimestamp: '2024-08-06T16:50:00.000Z',
-    definitionId: 'admin-console-services',
+    metrics: {},
+    definition_id: 'admin-console-services',
   },
 };
 
@@ -47,7 +43,7 @@ const metadata = {
     ingested: '2024-08-06T17:06:24.444700Z',
     category: '',
   },
-  sourceIndex: ['kbn-data-forge-fake_stack.message_processor-2024-08-01'],
+  source_index: ['kbn-data-forge-fake_stack.message_processor-2024-08-01'],
   log: {
     logger: 'message_processor',
   },

x-pack/packages/kbn-entities-schema/src/schema/entity.ts

@@ -11,12 +11,12 @@ import { arrayOfStringsSchema } from './common';
 export const entityBaseSchema = z.object({
   id: z.string(),
   type: z.string(),
-  identityFields: arrayOfStringsSchema,
-  displayName: z.string(),
+  identity_fields: arrayOfStringsSchema,
+  display_name: z.string(),
   metrics: z.record(z.string(), z.number()),
-  definitionVersion: z.string(),
-  schemaVersion: z.string(),
-  definitionId: z.string(),
+  definition_version: z.string(),
+  schema_version: z.string(),
+  definition_id: z.string(),
 });
 
 export interface MetadataRecord {
@@ -34,15 +34,8 @@ export const entityLatestSchema = z
   .object({
     entity: entityBaseSchema.merge(
       z.object({
-        lastSeenTimestamp: z.string(),
+        last_seen_timestamp: z.string(),
       })
     ),
   })
   .and(entityMetadataSchema);
-
-export const entityHistorySchema = z
-  .object({
-    '@timestamp': z.string(),
-    entity: entityBaseSchema,
-  })
-  .and(entityMetadataSchema);

x-pack/plugins/entity_manager/server/lib/auth/privileges.ts

@@ -59,7 +59,7 @@ export const entityDefinitionRuntimePrivileges = {
   index: [
     {
       names: [ENTITY_INTERNAL_INDICES_PATTERN],
-      privileges: ['create_index', 'index', 'create_doc', 'auto_configure', 'read'],
+      privileges: ['create_index', 'delete_index', 'index', 'create_doc', 'auto_configure', 'read'],
     },
     {
       names: [...BUILT_IN_ALLOWED_INDICES, ENTITY_INTERNAL_INDICES_PATTERN],

x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts

@@ -29,7 +29,7 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
     identityFields: ['service.name'],
     displayNameTemplate: '{{service.name}}',
     metadata: [
-      { source: '_index', destination: 'sourceIndex' },
+      { source: '_index', destination: 'source_index' },
       {
         source: 'data_stream.type',
         destination: 'source_data_stream.type',
@@ -38,7 +38,7 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
         source: 'data_stream.dataset',
         destination: 'source_data_stream.dataset',
       },
-      { source: 'agent.name', aggregation: { type: 'terms', limit: 100 } },
+      'agent.name',
       'service.environment',
       'service.name',
       'service.namespace',

x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts

@@ -25,7 +25,7 @@ export async function createAndInstallIngestPipelines(
           id: latestId,
           processors: latestProcessors,
           _meta: {
-            definitionVersion: definition.version,
+            definition_version: definition.version,
             managed: definition.managed,
           },
         }),