[Inventory] Inventory k8s entities fixes (#201260)

closes [#201226](#201226) ## Summary This PR makes the final adjustments on k8s entities after the #196916 was merged. I had to fix most of the ECS entities because they had their `entityId` defined with `uid` field. Most of the ECS K8s entities don't have this field and should use the `name` field instead (see [metricsets](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html#_metricsets_41)) ~I also had to fix the transforms to include an aggregation for the `displayNameTemplate` field, when it doesn't match with the `entity.Id`~ ### Real data Otel The screenshots below are from a tests running the `opentemeletry-demo` sending otel data <img width="710" alt="image" src="https://github.com/user-attachments/assets/d223858d-3b99-4bb0-b69e-3b70112a2c17"> <img width="710" alt="image" src="https://github.com/user-attachments/assets/1ef27dcd-682c-4681-b92b-be9b4f2b32b8"> ### Real data ECS The screenshots below are from a tests running the `opentemeletry-demo` with elastic agent installed with Kubernetes integration <img width="710" alt="image" src="https://github.com/user-attachments/assets/e2fb6cbd-60a0-4995-bc94-8ccbc8911db1"> <img width="710" alt="image" src="https://github.com/user-attachments/assets/a3d9dba3-fee7-42af-972b-34a151b52f2b"> ### Additional test with synthtrace <img width="709" alt="image" src="https://github.com/user-attachments/assets/bf56da1e-0bbb-40a0-9f44-06b9f86427a8"> ### Fix ECS k8s service entity was missing the link to its corresponding dashboard <img width="709" alt="image" src="https://github.com/user-attachments/assets/488891a6-1c61-4001-b604-208478e1c798"> <img width="709" alt="image" src="https://github.com/user-attachments/assets/0b30c56b-db52-4d82-a7e7-100dae91a35e"> ### How to test - Start local kibana and es instances - run ` node scripts/synthtrace k8s_entities.ts --clean --live` - Navigate to Inventory and enable EEM --------- Co-authored-by: Elastic Machine <[email protected]>
elastic · Nov 26, 2024 · aead7b9 · aead7b9
1 parent 3188cda
commit aead7b9
Show file tree

Hide file tree

Showing 42 changed files with 282 additions and 107 deletions.
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts
@@ -15,15 +15,16 @@ import { k8sClusterJobEntity } from './kubernetes/cluster_entity';
 import { k8sCronJobEntity } from './kubernetes/cron_job_entity';
 import { k8sDaemonSetEntity } from './kubernetes/daemon_set_entity';
 import { k8sDeploymentEntity } from './kubernetes/deployment_entity';
-import { k8sJobSetEntity } from './kubernetes/job_set_entity';
+import { k8sJobEntity } from './kubernetes/job_entity';
 import { k8sNodeEntity } from './kubernetes/node_entity';
 import { k8sPodEntity } from './kubernetes/pod_entity';
 import { k8sReplicaSetEntity } from './kubernetes/replica_set';
 import { k8sStatefulSetEntity } from './kubernetes/stateful_set';
+import { k8sServiceEntity } from './kubernetes/service';
 import { k8sContainerEntity } from './kubernetes/container_entity';
 
 export type EntityDataStreamType = 'metrics' | 'logs' | 'traces';
-export type Schema = 'ecs' | 'semconv';
+export type Schema = 'ecs' | 'otel';
 
 export type EntityFields = Fields &
   Partial<{
@@ -52,11 +53,12 @@ export const entities = {
     k8sCronJobEntity,
     k8sDaemonSetEntity,
     k8sDeploymentEntity,
-    k8sJobSetEntity,
+    k8sJobEntity,
     k8sNodeEntity,
     k8sPodEntity,
     k8sReplicaSetEntity,
     k8sStatefulSetEntity,
+    k8sServiceEntity,
     k8sContainerEntity,
   },
 };
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cluster_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cluster_entity.ts
@@ -23,6 +23,7 @@ export function k8sClusterJobEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
+      'entity.definition_id': 'cluster',
       'entity.type': 'cluster',
       'orchestrator.cluster.name': name,
       'entity.id': entityId,
@@ -31,6 +32,7 @@ export function k8sClusterJobEntity({
   }
 
   return new K8sEntity(schema, {
+    'entity.definition_id': 'cluster',
     'entity.type': 'cluster',
     'k8s.cluster.uid': name,
     'entity.id': entityId,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/container_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/container_entity.ts
@@ -23,6 +23,7 @@ export function k8sContainerEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
+      'entity.definition_id': 'container',
       'entity.type': 'container',
       'kubernetes.container.id': id,
       'entity.id': entityId,
@@ -31,6 +32,7 @@ export function k8sContainerEntity({
   }
 
   return new K8sEntity(schema, {
+    'entity.definition_id': 'container',
     'entity.type': 'container',
     'container.id': id,
     'entity.id': entityId,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cron_job_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cron_job_entity.ts
@@ -27,17 +27,18 @@ export function k8sCronJobEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
-      'entity.type': 'cron_job',
+      'entity.definition_id': 'cron_job',
+      'entity.type': 'cronjob',
       'kubernetes.cronjob.name': name,
-      'kubernetes.cronjob.uid': uid,
       'kubernetes.namespace': clusterName,
       'entity.id': entityId,
       ...others,
     });
   }
 
   return new K8sEntity(schema, {
-    'entity.type': 'cron_job',
+    'entity.definition_id': 'cron_job',
+    'entity.type': 'cronjob',
     'k8s.cronjob.name': name,
     'k8s.cronjob.uid': uid,
     'k8s.cluster.name': clusterName,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/daemon_set_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/daemon_set_entity.ts
@@ -27,7 +27,8 @@ export function k8sDaemonSetEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
-      'entity.type': 'daemon_set',
+      'entity.definition_id': 'daemon_set',
+      'entity.type': 'daemonset',
       'kubernetes.daemonset.name': name,
       'kubernetes.daemonset.uid': uid,
       'kubernetes.namespace': clusterName,
@@ -37,7 +38,8 @@ export function k8sDaemonSetEntity({
   }
 
   return new K8sEntity(schema, {
-    'entity.type': 'daemon_set',
+    'entity.definition_id': 'daemon_set',
+    'entity.type': 'daemonset',
     'k8s.daemonset.name': name,
     'k8s.daemonset.uid': uid,
     'k8s.cluster.name': clusterName,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/deployment_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/deployment_entity.ts
@@ -27,16 +27,17 @@ export function k8sDeploymentEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
+      'entity.definition_id': 'deployment',
       'entity.type': 'deployment',
       'kubernetes.deployment.name': name,
-      'kubernetes.deployment.uid': uid,
       'kubernetes.namespace': clusterName,
       'entity.id': entityId,
       ...others,
     });
   }
 
   return new K8sEntity(schema, {
+    'entity.definition_id': 'deployment',
     'entity.type': 'deployment',
     'k8s.deployment.name': name,
     'k8s.deployment.uid': uid,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts
@@ -12,27 +12,28 @@ import { Serializable } from '../../serializable';
 
 const identityFieldsMap: Record<Schema, Record<string, string[]>> = {
   ecs: {
-    pod: ['kubernetes.pod.name'],
+    pod: ['kubernetes.pod.uid'],
     cluster: ['orchestrator.cluster.name'],
-    cron_job: ['kubernetes.cronjob.name'],
-    daemon_set: ['kubernetes.daemonset.name'],
+    cronjob: ['kubernetes.cronjob.name'],
+    daemonset: ['kubernetes.daemonset.name'],
     deployment: ['kubernetes.deployment.name'],
     job: ['kubernetes.job.name'],
     node: ['kubernetes.node.name'],
-    replica_set: ['kubernetes.replicaset.name'],
-    stateful_set: ['kubernetes.statefulset.name'],
+    replicaset: ['kubernetes.replicaset.name'],
+    statefulset: ['kubernetes.statefulset.name'],
+    service: ['kubernetes.service.name'],
     container: ['kubernetes.container.id'],
   },
-  semconv: {
-    pod: ['k8s.pod.name'],
+  otel: {
+    pod: ['k8s.pod.uid'],
     cluster: ['k8s.cluster.uid'],
-    cron_job: ['k8s.cronjob.name'],
-    daemon_set: ['k8s.daemonset.name'],
-    deployment: ['k8s.deployment.name'],
-    job: ['k8s.job.name'],
+    cronjob: ['k8s.cronjob.uid'],
+    daemonset: ['k8s.daemonset.uid'],
+    deployment: ['k8s.deployment.uid'],
+    job: ['k8s.job.uid'],
     node: ['k8s.node.uid'],
-    replica_set: ['k8s.replicaset.name'],
-    stateful_set: ['k8s.statefulset.name'],
+    replicaset: ['k8s.replicaset.uid'],
+    statefulset: ['k8s.statefulset.uid'],
     container: ['container.id'],
   },
 };
@@ -41,10 +42,17 @@ export class K8sEntity extends Serializable<EntityFields> {
   constructor(schema: Schema, fields: EntityFields) {
     const entityType = fields['entity.type'];
     if (entityType === undefined) {
-      throw new Error(`Entity type not defined: ${entityType}`);
+      throw new Error(`Entity type not defined`);
     }
 
-    const entityTypeWithSchema = `kubernetes_${entityType}_${schema}`;
+    const entityDefinitionId = fields['entity.definition_id'];
+    if (entityDefinitionId === undefined) {
+      throw new Error(`Entity definition id not defined`);
+    }
+
+    const entityDefinitionWithSchema = `kubernetes_${entityDefinitionId}_${
+      schema === 'ecs' ? schema : 'semconv'
+    }`;
     const identityFields = identityFieldsMap[schema][entityType];
     if (identityFields === undefined || identityFields.length === 0) {
       throw new Error(
@@ -54,8 +62,8 @@ export class K8sEntity extends Serializable<EntityFields> {
 
     super({
       ...fields,
-      'entity.type': entityTypeWithSchema,
-      'entity.definition_id': `builtin_${entityTypeWithSchema}`,
+      'entity.type': `k8s.${entityType}.${schema}`,
+      'entity.definition_id': `builtin_${entityDefinitionWithSchema}`,
       'entity.identity_fields': identityFields,
       'entity.display_name': getDisplayName({ identityFields, fields }),
       'entity.definition_version': '1.0.0',

diff --git a/...lib/entities/kubernetes/job_set_entity.ts → ...src/lib/entities/kubernetes/job_entity.ts b/...lib/entities/kubernetes/job_set_entity.ts → ...src/lib/entities/kubernetes/job_entity.ts
@@ -10,7 +10,7 @@
 import { Schema } from '..';
 import { K8sEntity } from '.';
 
-export function k8sJobSetEntity({
+export function k8sJobEntity({
   schema,
   name,
   uid,
@@ -27,16 +27,17 @@ export function k8sJobSetEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
+      'entity.definition_id': 'job',
       'entity.type': 'job',
       'kubernetes.job.name': name,
-      'kubernetes.job.uid': uid,
       'kubernetes.namespace': clusterName,
       'entity.id': entityId,
       ...others,
     });
   }
 
   return new K8sEntity(schema, {
+    'entity.definition_id': 'job',
     'entity.type': 'job',
     'k8s.job.name': name,
     'k8s.job.uid': uid,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/node_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/node_entity.ts
@@ -27,16 +27,17 @@ export function k8sNodeEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
+      'entity.definition_id': 'node',
       'entity.type': 'node',
       'kubernetes.node.name': name,
-      'kubernetes.node.uid': uid,
       'kubernetes.namespace': clusterName,
       'entity.id': entityId,
       ...others,
     });
   }
 
   return new K8sEntity(schema, {
+    'entity.definition_id': 'node',
     'entity.type': 'node',
     'k8s.node.uid': uid,
     'k8s.cluster.name': clusterName,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/pod_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/pod_entity.ts
@@ -27,6 +27,7 @@ export function k8sPodEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
+      'entity.definition_id': 'pod',
       'entity.type': 'pod',
       'kubernetes.pod.name': name,
       'kubernetes.pod.uid': uid,
@@ -37,6 +38,7 @@ export function k8sPodEntity({
   }
 
   return new K8sEntity(schema, {
+    'entity.definition_id': 'pod',
     'entity.type': 'pod',
     'k8s.pod.name': name,
     'k8s.pod.uid': uid,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/replica_set.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/replica_set.ts
@@ -27,17 +27,18 @@ export function k8sReplicaSetEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
-      'entity.type': 'replica_set',
+      'entity.definition_id': 'replica_set',
+      'entity.type': 'replicaset',
       'kubernetes.replicaset.name': name,
-      'kubernetes.replicaset.uid': uid,
       'kubernetes.namespace': clusterName,
       'entity.id': entityId,
       ...others,
     });
   }
 
   return new K8sEntity(schema, {
-    'entity.type': 'replica_set',
+    'entity.definition_id': 'replica_set',
+    'entity.type': 'replicaset',
     'k8s.replicaset.name': name,
     'k8s.replicaset.uid': uid,
     'k8s.cluster.name': clusterName,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/service.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/service.ts
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Schema } from '..';
+import { K8sEntity } from '.';
+
+export function k8sServiceEntity({
+  schema,
+  name,
+  uid,
+  clusterName,
+  entityId,
+  ...others
+}: {
+  schema: Schema;
+  name: string;
+  uid?: string;
+  clusterName?: string;
+  entityId: string;
+  [key: string]: any;
+}) {
+  if (schema !== 'ecs') {
+    throw new Error('Schema not supported for service entity: ' + schema);
+  }
+  return new K8sEntity(schema, {
+    'entity.definition_id': 'service',
+    'entity.type': 'service',
+    'kubernetes.service.name': name,
+    'kubernetes.namespace': clusterName,
+    'entity.id': entityId,
+    ...others,
+  });
+}
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/stateful_set.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/stateful_set.ts
@@ -27,17 +27,18 @@ export function k8sStatefulSetEntity({
 }) {
   if (schema === 'ecs') {
     return new K8sEntity(schema, {
-      'entity.type': 'stateful_set',
+      'entity.definition_id': 'stateful_set',
+      'entity.type': 'statefulset',
       'kubernetes.statefulset.name': name,
-      'kubernetes.statefulset.uid': uid,
       'kubernetes.namespace': clusterName,
       'entity.id': entityId,
       ...others,
     });
   }
 
   return new K8sEntity(schema, {
-    'entity.type': 'stateful_set',
+    'entity.definition_id': 'stateful_set',
+    'entity.type': 'statefulset',
     'k8s.statefulset.name': name,
     'k8s.statefulset.uid': uid,
     'k8s.cluster.name': clusterName,

diff --git a/packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts
@@ -75,15 +75,13 @@ function getRoutingTransform() {
   return new Transform({
     objectMode: true,
     transform(document: ESDocumentWithOperation<EntityFields>, encoding, callback) {
-      const entityType: string | undefined = document['entity.type'];
-      if (entityType === undefined) {
-        throw new Error(`entity.type was not defined: ${JSON.stringify(document)}`);
+      const definitionId: string | undefined = document['entity.definition_id'];
+      if (definitionId === undefined) {
+        throw new Error(`entity.definition_id was not defined: ${JSON.stringify(document)}`);
       }
-      const entityIndexName = `${entityType}s`;
       document._action = {
         index: {
-          _index:
-            `.entities.v1.latest.builtin_${entityIndexName}_from_ecs_data`.toLocaleLowerCase(),
+          _index: `.entities.v1.latest.${definitionId}`.toLocaleLowerCase(),
           _id: document['entity.id'],
         },
       };