[Inventory] List k8s entities in the grid (#197292)

closes #196155
Blocked by #196916 (K8s entities
alias patterns don't exist yet.)

```
node scripts/synthtrace many_entities.ts --clean --live
node scripts/synthtrace k8s_entities.ts --clean --live
```

https://github.com/user-attachments/assets/5861ebc7-8386-4a4b-a68b-50adc5244d43
(cherry picked from commit c6f4178)

# Conflicts:
#	x-pack/plugins/observability_solution/inventory/public/components/search_bar/index.tsx

packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts

@@ -0,0 +1,185 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityFields, entities, generateShortId } from '@kbn/apm-synthtrace-client';
+import { Schema } from '@kbn/apm-synthtrace-client/src/lib/entities';
+import { Scenario } from '../cli/scenario';
+import { withClient } from '../lib/utils/with_client';
+
+const CLUSTER_NAME = 'cluster_foo';
+
+const CLUSTER_ENTITY_ID = generateShortId();
+const POD_ENTITY_ID = generateShortId();
+const POD_UID = generateShortId();
+const REPLICA_SET_ENTITY_ID = generateShortId();
+const REPLICA_SET_UID = generateShortId();
+const DEPLOYMENT_ENTITY_ID = generateShortId();
+const DEPLOYMENT_UID = generateShortId();
+const STATEFUL_SET_ENTITY_ID = generateShortId();
+const STATEFUL_SET_UID = generateShortId();
+const DAEMON_SET_ENTITY_ID = generateShortId();
+const DAEMON_SET_UID = generateShortId();
+const JOB_SET_ENTITY_ID = generateShortId();
+const JOB_SET_UID = generateShortId();
+const CRON_JOB_ENTITY_ID = generateShortId();
+const CRON_JOB_UID = generateShortId();
+const NODE_ENTITY_ID = generateShortId();
+const NODE_UID = generateShortId();
+const SYNTH_JAVA_TRACE_ENTITY_ID = generateShortId();
+const SYNTH_HOST_FOO_LOGS_ENTITY_ID = generateShortId();
+const SYNTH_CONTAINER_FOO_LOGS_ENTITY_ID = generateShortId();
+
+const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
+  const { logger } = runOptions;
+
+  return {
+    bootstrap: async ({ entitiesKibanaClient }) => {
+      await entitiesKibanaClient.installEntityIndexPatterns();
+    },
+    generate: ({ range, clients: { entitiesEsClient } }) => {
+      const rangeInterval = range.interval('1m').rate(1);
+      const getK8sEntitiesEvents = (schema: Schema) =>
+        rangeInterval.generator((timestamp) => {
+          return [
+            entities.k8s
+              .k8sClusterJobEntity({
+                schema,
+                name: CLUSTER_NAME,
+                entityId: CLUSTER_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sPodEntity({
+                schema,
+                clusterName: CLUSTER_NAME,
+                name: 'pod_foo',
+                uid: POD_UID,
+                entityId: POD_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sReplicaSetEntity({
+                clusterName: CLUSTER_NAME,
+                name: 'replica_set_foo',
+                schema,
+                uid: REPLICA_SET_UID,
+                entityId: REPLICA_SET_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sDeploymentEntity({
+                clusterName: CLUSTER_NAME,
+                name: 'deployment_foo',
+                schema,
+                uid: DEPLOYMENT_UID,
+                entityId: DEPLOYMENT_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sStatefulSetEntity({
+                clusterName: CLUSTER_NAME,
+                name: 'stateful_set_foo',
+                schema,
+                uid: STATEFUL_SET_UID,
+                entityId: STATEFUL_SET_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sDaemonSetEntity({
+                clusterName: CLUSTER_NAME,
+                name: 'daemon_set_foo',
+                schema,
+                uid: DAEMON_SET_UID,
+                entityId: DAEMON_SET_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sJobSetEntity({
+                clusterName: CLUSTER_NAME,
+                name: 'job_set_foo',
+                schema,
+                uid: JOB_SET_UID,
+                entityId: JOB_SET_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sCronJobEntity({
+                clusterName: CLUSTER_NAME,
+                name: 'cron_job_foo',
+                schema,
+                uid: CRON_JOB_UID,
+                entityId: CRON_JOB_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sNodeEntity({
+                clusterName: CLUSTER_NAME,
+                name: 'node_job_foo',
+                schema,
+                uid: NODE_UID,
+                entityId: NODE_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+            entities.k8s
+              .k8sContainerEntity({
+                id: '123',
+                schema,
+                entityId: NODE_ENTITY_ID,
+              })
+              .timestamp(timestamp),
+          ];
+        });
+
+      const ecsEntities = getK8sEntitiesEvents('ecs');
+      const otelEntities = getK8sEntitiesEvents('semconv');
+      const synthJavaTraces = entities.serviceEntity({
+        serviceName: 'synth_java',
+        agentName: ['java'],
+        dataStreamType: ['traces'],
+        environment: 'production',
+        entityId: SYNTH_JAVA_TRACE_ENTITY_ID,
+      });
+      const synthHostFooLogs = entities.hostEntity({
+        hostName: 'synth_host_foo',
+        agentName: ['macbook'],
+        dataStreamType: ['logs'],
+        entityId: SYNTH_HOST_FOO_LOGS_ENTITY_ID,
+      });
+      const synthContainerFooLogs = entities.containerEntity({
+        containerId: 'synth_container_foo',
+        agentName: ['macbook'],
+        dataStreamType: ['logs'],
+        entityId: SYNTH_CONTAINER_FOO_LOGS_ENTITY_ID,
+      });
+
+      const otherEvents = rangeInterval.generator((timestamp) => [
+        synthJavaTraces.timestamp(timestamp),
+        synthHostFooLogs.timestamp(timestamp),
+        synthContainerFooLogs.timestamp(timestamp),
+      ]);
+
+      return [
+        withClient(
+          entitiesEsClient,
+          logger.perf('generating_entities_k8s_ecs_events', () => ecsEntities)
+        ),
+        withClient(
+          entitiesEsClient,
+          logger.perf('generating_entities_k8s_otel_events', () => otelEntities)
+        ),
+        withClient(
+          entitiesEsClient,
+          logger.perf('generating_entities_other_events', () => otherEvents)
+        ),
+      ];
+    },
+  };
+};
+
+export default scenario;

x-pack/plugins/observability_solution/inventory/common/entities.ts

@@ -6,12 +6,6 @@
  */
 import { ENTITY_LATEST, entitiesAliasPattern } from '@kbn/entities-schema';
 import {
-  HOST_NAME,
-  SERVICE_ENVIRONMENT,
-  SERVICE_NAME,
-  AGENT_NAME,
-  CLOUD_PROVIDER,
-  CONTAINER_ID,
   ENTITY_DEFINITION_ID,
   ENTITY_DISPLAY_NAME,
   ENTITY_ID,
@@ -22,12 +16,6 @@ import {
 import { isRight } from 'fp-ts/lib/Either';
 import * as t from 'io-ts';
 
-export const entityTypeRt = t.union([
-  t.literal('service'),
-  t.literal('host'),
-  t.literal('container'),
-]);
-
 export const entityColumnIdsRt = t.union([
   t.literal(ENTITY_DISPLAY_NAME),
   t.literal(ENTITY_LAST_SEEN),
@@ -37,8 +25,6 @@ export const entityColumnIdsRt = t.union([
 
 export type EntityColumnIds = t.TypeOf<typeof entityColumnIdsRt>;
 
-export type EntityType = t.TypeOf<typeof entityTypeRt>;
-
 export const defaultEntitySortField: EntityColumnIds = 'alertsCount';
 
 export const MAX_NUMBER_OF_ENTITIES = 500;
@@ -48,20 +34,8 @@ export const ENTITIES_LATEST_ALIAS = entitiesAliasPattern({
   dataset: ENTITY_LATEST,
 });
 
-const BUILTIN_SERVICES_FROM_ECS_DATA = 'builtin_services_from_ecs_data';
-const BUILTIN_HOSTS_FROM_ECS_DATA = 'builtin_hosts_from_ecs_data';
-const BUILTIN_CONTAINERS_FROM_ECS_DATA = 'builtin_containers_from_ecs_data';
-
-export const defaultEntityDefinitions = [
-  BUILTIN_SERVICES_FROM_ECS_DATA,
-  BUILTIN_HOSTS_FROM_ECS_DATA,
-  BUILTIN_CONTAINERS_FROM_ECS_DATA,
-];
-
-export const defaultEntityTypes: EntityType[] = ['service', 'host', 'container'];
-
-const entityArrayRt = t.array(entityTypeRt);
-export const entityTypesRt = new t.Type<EntityType[], string, unknown>(
+const entityArrayRt = t.array(t.string);
+export const entityTypesRt = new t.Type<string[], string, unknown>(
   'entityTypesRt',
   entityArrayRt.is,
   (input, context) => {
@@ -83,37 +57,13 @@ export const entityTypesRt = new t.Type<EntityType[], string, unknown>(
   (arr) => arr.join()
 );
 
-interface BaseEntity {
+export interface Entity {
   [ENTITY_LAST_SEEN]: string;
   [ENTITY_ID]: string;
-  [ENTITY_TYPE]: EntityType;
+  [ENTITY_TYPE]: string;
   [ENTITY_DISPLAY_NAME]: string;
   [ENTITY_DEFINITION_ID]: string;
   [ENTITY_IDENTITY_FIELDS]: string | string[];
   alertsCount?: number;
   [key: string]: any;
 }
-
-/**
- * These types are based on service, host and container from the built in definition.
- */
-export interface ServiceEntity extends BaseEntity {
-  [ENTITY_TYPE]: 'service';
-  [SERVICE_NAME]: string;
-  [SERVICE_ENVIRONMENT]?: string | string[] | null;
-  [AGENT_NAME]: string | string[] | null;
-}
-
-export interface HostEntity extends BaseEntity {
-  [ENTITY_TYPE]: 'host';
-  [HOST_NAME]: string;
-  [CLOUD_PROVIDER]: string | string[] | null;
-}
-
-export interface ContainerEntity extends BaseEntity {
-  [ENTITY_TYPE]: 'container';
-  [CONTAINER_ID]: string;
-  [CLOUD_PROVIDER]: string | string[] | null;
-}
-
-export type Entity = ServiceEntity | HostEntity | ContainerEntity;

x-pack/plugins/observability_solution/inventory/common/entitites.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 import { isLeft, isRight } from 'fp-ts/lib/Either';
-import { type EntityType, entityTypesRt } from './entities';
+import { entityTypesRt } from './entities';
 
 const validate = (input: unknown) => entityTypesRt.decode(input);
 
@@ -28,36 +28,12 @@ describe('entityTypesRt codec', () => {
     }
   });
 
-  it('should fail validation when the string contains invalid entity types', () => {
-    const input = 'service,invalidType,host';
-    const result = validate(input);
-    expect(isLeft(result)).toBe(true);
-  });
-
-  it('should fail validation when the array contains invalid entity types', () => {
-    const input = ['service', 'invalidType', 'host'];
-    const result = validate(input);
-    expect(isLeft(result)).toBe(true);
-  });
-
   it('should fail validation when input is not a string or array', () => {
     const input = 123;
     const result = validate(input);
     expect(isLeft(result)).toBe(true);
   });
 
-  it('should fail validation when the array contains non-string elements', () => {
-    const input = ['service', 123, 'host'];
-    const result = validate(input);
-    expect(isLeft(result)).toBe(true);
-  });
-
-  it('should fail validation an empty string', () => {
-    const input = '';
-    const result = validate(input);
-    expect(isLeft(result)).toBe(true);
-  });
-
   it('should validate an empty array as valid', () => {
     const input: unknown[] = [];
     const result = validate(input);
@@ -67,32 +43,14 @@ describe('entityTypesRt codec', () => {
     }
   });
 
-  it('should fail validation when the string contains only commas', () => {
-    const input = ',,,';
-    const result = validate(input);
-    expect(isLeft(result)).toBe(true);
-  });
-
-  it('should fail validation for partial valid entities in a string', () => {
-    const input = 'service,invalidType';
-    const result = validate(input);
-    expect(isLeft(result)).toBe(true);
-  });
-
-  it('should fail validation for partial valid entities in an array', () => {
-    const input = ['service', 'invalidType'];
-    const result = validate(input);
-    expect(isLeft(result)).toBe(true);
-  });
-
   it('should serialize a valid array back to a string', () => {
-    const input: EntityType[] = ['service', 'host'];
+    const input = ['service', 'host'];
     const serialized = entityTypesRt.encode(input);
     expect(serialized).toBe('service,host');
   });
 
   it('should serialize an empty array back to an empty string', () => {
-    const input: EntityType[] = [];
+    const input: string[] = [];
     const serialized = entityTypesRt.encode(input);
     expect(serialized).toBe('');
   });