[Inventory][ECO] Replace Entity with InventoryEntityLatest type

packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts

@@ -58,6 +58,8 @@ export class K8sEntity extends Serializable<EntityFields> {
       'entity.definition_id': `builtin_${entityTypeWithSchema}`,
       'entity.identity_fields': identityFields,
       'entity.display_name': getDisplayName({ identityFields, fields }),
+      'entity.definition_version': '1.0.0',
+      'entity.schema_version': '1.0',
     });
   }
 }

x-pack/packages/kbn-entities-schema/src/schema/entity.ts

@@ -11,9 +11,9 @@ import { arrayOfStringsSchema } from './common';
 export const entityBaseSchema = z.object({
   id: z.string(),
   type: z.string(),
-  identity_fields: arrayOfStringsSchema,
+  identity_fields: z.union([arrayOfStringsSchema, z.string()]),
   display_name: z.string(),
-  metrics: z.record(z.string(), z.number()),
+  metrics: z.optional(z.record(z.string(), z.number())),
   definition_version: z.string(),
   schema_version: z.string(),
   definition_id: z.string(),
@@ -24,10 +24,18 @@ export interface MetadataRecord {
 }
 
 const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
+
 type Literal = z.infer<typeof literalSchema>;
-type Metadata = Literal | { [key: string]: Metadata } | Metadata[];
-export const entityMetadataSchema: z.ZodType<Metadata> = z.lazy(() =>
-  z.union([literalSchema, z.array(entityMetadataSchema), z.record(entityMetadataSchema)])
+interface Metadata {
+  [key: string]: Literal | Literal[] | Metadata;
+}
+
+// Define the recursive schema using z.lazy
+export const entityMetadataSchema: z.ZodSchema<Metadata> = z.lazy(() =>
+  z.union([
+    z.record(literalSchema), // Object with string keys and values of type string, number, or boolean
+    entityMetadataSchema, // Recursive reference to SomeType
+  ])
 );
 
 export const entityLatestSchema = z

x-pack/packages/observability/observability_utils/array/join_by_key.test.ts

@@ -221,4 +221,50 @@ describe('joinByKey', () => {
       },
     });
   });
+
+  it('deeply merges by unflatten keys', () => {
+    const joined = joinByKey(
+      [
+        {
+          service: {
+            name: 'opbeans-node',
+            metrics: {
+              cpu: 0.1,
+            },
+          },
+          properties: {
+            foo: 'bar',
+          },
+        },
+        {
+          service: {
+            environment: 'prod',
+            metrics: {
+              memory: 0.5,
+            },
+          },
+          properties: {
+            foo: 'bar',
+          },
+        },
+      ],
+      'properties.foo'
+    );
+
+    expect(joined).toEqual([
+      {
+        service: {
+          name: 'opbeans-node',
+          environment: 'prod',
+          metrics: {
+            cpu: 0.1,
+            memory: 0.5,
+          },
+        },
+        properties: {
+          foo: 'bar',
+        },
+      },
+    ]);
+  });
 });

x-pack/packages/observability/observability_utils/array/join_by_key.ts

@@ -18,18 +18,29 @@ export type JoinedReturnType<
   }
 >;
 
-type ArrayOrSingle<T> = T | T[];
+function getValueByPath(obj: any, path: string): any {
+  return path.split('.').reduce((acc, keyPart) => {
+    // Check if acc is a valid object and has the key
+    return acc && acc[keyPart] !== undefined ? acc[keyPart] : undefined;
+  }, obj);
+}
 
+type NestedKeys<T> = T extends object
+  ? { [K in keyof T]: K extends string ? `${K}` | `${K}.${NestedKeys<T[K]>}` : never }[keyof T]
+  : never;
+
+type ArrayOrSingle<T> = T | T[];
+type CombinedNestedKeys<T, U> = (NestedKeys<T> & NestedKeys<U>) | (keyof T & keyof U);
 export function joinByKey<
   T extends Record<string, any>,
   U extends UnionToIntersection<T>,
-  V extends ArrayOrSingle<keyof T & keyof U>
+  V extends ArrayOrSingle<CombinedNestedKeys<T, U>>
 >(items: T[], key: V): JoinedReturnType<T, U>;
 
 export function joinByKey<
   T extends Record<string, any>,
   U extends UnionToIntersection<T>,
-  V extends ArrayOrSingle<keyof T & keyof U>,
+  V extends ArrayOrSingle<CombinedNestedKeys<T, U>>,
   W extends JoinedReturnType<T, U>,
   X extends (a: T, b: T) => ValuesType<W>
 >(items: T[], key: V, mergeFn: X): W;
@@ -45,7 +56,7 @@ export function joinByKey(
   items.forEach((current) => {
     // The key of the map is a stable JSON string of the values from given keys.
     // We need stable JSON string to support plain object values.
-    const stableKey = stableStringify(keys.map((k) => current[k]));
+    const stableKey = stableStringify(keys.map((k) => current[k] ?? getValueByPath(current, k)));
 
     if (map.has(stableKey)) {
       const item = map.get(stableKey);

x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts

@@ -9,6 +9,7 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server';
 import type { ESQLSearchResponse, ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import { withSpan } from '@kbn/apm-utils';
 import type { EsqlQueryRequest } from '@elastic/elasticsearch/lib/api/types';
+import { esqlResultToPlainObjects } from '../utils/esql_result_to_plain_objects';
 
 type SearchRequest = ESSearchRequest & {
   index: string | string[];
@@ -25,7 +26,10 @@ export interface ObservabilityElasticsearchClient {
     operationName: string,
     parameters: TSearchRequest
   ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
-  esql(operationName: string, parameters: EsqlQueryRequest): Promise<ESQLSearchResponse>;
+  esql<TDocument = unknown>(
+    operationName: string,
+    parameters: EsqlQueryRequest
+  ): Promise<TDocument[]>;
   client: ElasticsearchClient;
 }
 
@@ -40,7 +44,7 @@ export function createObservabilityEsClient({
 }): ObservabilityElasticsearchClient {
   return {
     client,
-    esql(operationName: string, parameters: EsqlQueryRequest) {
+    esql<TDocument = unknown>(operationName: string, parameters: EsqlQueryRequest) {
       logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`);
       return withSpan({ name: operationName, labels: { plugin } }, () => {
         return client.esql.query(
@@ -54,7 +58,7 @@ export function createObservabilityEsClient({
       })
         .then((response) => {
           logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`);
-          return response as unknown as ESQLSearchResponse;
+          return esqlResultToPlainObjects<TDocument>(response as unknown as ESQLSearchResponse);
         })
         .catch((error) => {
           throw error;

x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts

@@ -6,25 +6,28 @@
  */
 
 import type { ESQLSearchResponse } from '@kbn/es-types';
+import { unflattenObject } from '../../object/unflatten_object';
 
-export function esqlResultToPlainObjects<T extends Record<string, any>>(
+export function esqlResultToPlainObjects<TDocument = unknown>(
   result: ESQLSearchResponse
-): T[] {
+): TDocument[] {
   return result.values.map((row) => {
-    return row.reduce<Record<string, unknown>>((acc, value, index) => {
-      const column = result.columns[index];
+    return unflattenObject(
+      row.reduce<Record<string, any>>((acc, value, index) => {
+        const column = result.columns[index];
 
-      if (!column) {
-        return acc;
-      }
+        if (!column) {
+          return acc;
+        }
 
-      // Removes the type suffix from the column name
-      const name = column.name.replace(/\.(text|keyword)$/, '');
-      if (!acc[name]) {
-        acc[name] = value;
-      }
+        // Removes the type suffix from the column name
+        const name = column.name.replace(/\.(text|keyword)$/, '');
+        if (!acc[name]) {
+          acc[name] = value;
+        }
 
-      return acc;
-    }, {});
-  }) as T[];
+        return acc;
+      }, {})
+    ) as TDocument;
+  });
 }

x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts

@@ -7,11 +7,9 @@
 
 import { type EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client';
 import { findInventoryFields } from '@kbn/metrics-data-access-plugin/common';
-import {
-  EntityDataStreamType,
-  SOURCE_DATA_STREAM_TYPE,
-} from '@kbn/observability-shared-plugin/common';
+import { EntityDataStreamType } from '@kbn/observability-shared-plugin/common';
 import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client';
+import { castArray } from 'lodash';
 import { type InfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client';
 import { getHasMetricsData } from './get_has_metrics_data';
 import { getLatestEntity } from './get_latest_entity';
@@ -45,15 +43,15 @@ export async function getDataStreamTypes({
     return Array.from(sourceDataStreams);
   }
 
-  const entity = await getLatestEntity({
+  const latestEntity = await getLatestEntity({
     inventoryEsClient: obsEsClient,
     entityId,
     entityType,
     entityManagerClient,
   });
 
-  if (entity?.[SOURCE_DATA_STREAM_TYPE]) {
-    [entity[SOURCE_DATA_STREAM_TYPE]].flat().forEach((item) => {
+  if (latestEntity) {
+    castArray(latestEntity.source_data_stream.type).forEach((item) => {
       sourceDataStreams.add(item as EntityDataStreamType);
     });
   }

x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts

@@ -12,15 +12,14 @@ import {
   SOURCE_DATA_STREAM_TYPE,
 } from '@kbn/observability-shared-plugin/common/field_names/elasticsearch';
 import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client';
-import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects';
 
 const ENTITIES_LATEST_ALIAS = entitiesAliasPattern({
   type: '*',
   dataset: ENTITY_LATEST,
 });
 
-interface Entity {
-  [SOURCE_DATA_STREAM_TYPE]: string | string[];
+interface LatestEntityResponse {
+  source_data_stream: { type?: string | string[] };
 }
 
 export async function getLatestEntity({
@@ -33,18 +32,18 @@ export async function getLatestEntity({
   entityType: 'host' | 'container';
   entityId: string;
   entityManagerClient: EntityClient;
-}): Promise<Entity | undefined> {
+}): Promise<LatestEntityResponse | undefined> {
   const { definitions } = await entityManagerClient.getEntityDefinitions({
     builtIn: true,
     type: entityType,
   });
 
   const hostOrContainerIdentityField = definitions[0]?.identityFields?.[0]?.field;
   if (hostOrContainerIdentityField === undefined) {
-    return { [SOURCE_DATA_STREAM_TYPE]: [] };
+    return undefined;
   }
 
-  const latestEntitiesEsqlResponse = await inventoryEsClient.esql('get_latest_entities', {
+  const response = await inventoryEsClient.esql<LatestEntityResponse>('get_latest_entities', {
     query: `FROM ${ENTITIES_LATEST_ALIAS}
         | WHERE ${ENTITY_TYPE} == ?
         | WHERE ${hostOrContainerIdentityField} == ?
@@ -53,5 +52,5 @@ export async function getLatestEntity({
     params: [entityType, entityId],
   });
 
-  return esqlResultToPlainObjects<Entity>(latestEntitiesEsqlResponse)[0];
+  return response[0];
 }

x-pack/plugins/observability_solution/inventory/.storybook/get_mock_inventory_context.tsx

@@ -24,7 +24,14 @@ export function getMockInventoryContext(): InventoryKibanaContext {
 
   return {
     ...coreStart,
-    entityManager: {} as unknown as EntityManagerPublicPluginStart,
+    entityManager: {
+      entityClient: {
+        asKqlFilter: jest.fn(),
+        getIdentityFieldsValue() {
+          return 'entity_id';
+        },
+      },
+    } as unknown as EntityManagerPublicPluginStart,
     observabilityShared: {} as unknown as ObservabilitySharedPluginStart,
     inference: {} as unknown as InferencePublicStart,
     share: {

x-pack/plugins/observability_solution/inventory/common/entities.ts

@@ -5,23 +5,20 @@
  * 2.0.
  */
 import { z } from '@kbn/zod';
-import { ENTITY_LATEST, entitiesAliasPattern, entityLatestSchema } from '@kbn/entities-schema';
 import {
-  ENTITY_DEFINITION_ID,
-  ENTITY_DISPLAY_NAME,
-  ENTITY_ID,
-  ENTITY_IDENTITY_FIELDS,
-  ENTITY_LAST_SEEN,
-  ENTITY_TYPE,
-} from '@kbn/observability-shared-plugin/common';
+  ENTITY_LATEST,
+  entitiesAliasPattern,
+  entityLatestSchema,
+  entityMetadataSchema,
+} from '@kbn/entities-schema';
 import { decode, encode } from '@kbn/rison';
 import { isRight } from 'fp-ts/lib/Either';
 import * as t from 'io-ts';
 
 export const entityColumnIdsRt = t.union([
-  t.literal(ENTITY_DISPLAY_NAME),
-  t.literal(ENTITY_LAST_SEEN),
-  t.literal(ENTITY_TYPE),
+  t.literal('entityDisplayName'),
+  t.literal('entityLastSeenTimestamp'),
+  t.literal('entityType'),
   t.literal('alertsCount'),
 ]);
 
@@ -102,23 +99,22 @@ export const entityTypesRt = new t.Type<string[], string, unknown>(
   (arr) => arr.join()
 );
 
-export interface Entity {
-  [ENTITY_LAST_SEEN]: string;
-  [ENTITY_ID]: string;
-  [ENTITY_TYPE]: string;
-  [ENTITY_DISPLAY_NAME]: string;
-  [ENTITY_DEFINITION_ID]: string;
-  [ENTITY_IDENTITY_FIELDS]: string | string[];
-  alertsCount?: number;
-  [key: string]: any;
-}
-
 export type EntityGroup = {
   count: number;
 } & {
   [key: string]: any;
 };
 
-export type InventoryEntityLatest = z.infer<typeof entityLatestSchema> & {
+export type EntityLatest = z.infer<typeof entityLatestSchema>;
+
+export type InventoryEntityLatest = {
+  entityId: string;
+  entityType: string;
+  entityIdentityFields: string | string[];
+  entityDisplayName: string;
+  entityDefinitionId: string;
+  entityLastSeenTimestamp: string;
+  entityDefinitionVersion: string;
+  entitySchemaVersion: string;
   alertsCount?: number;
-};
+} & z.infer<typeof entityMetadataSchema>;