[Infra][ECO] Adding summary API (elastic#194612)

closes elastic#193701 This PR does a few things: - Adds a new API endpoint on infra to fetch an entity summary: `/api/infra/entities/{entityType}/{entityId}/summary`. It fetches the latest EEM index filtering by entity.type(host | container) and depending on the entity type host.name or container.id. And it returns the following payload: ``` { "sourceDataStreams": [ "logs", "metrics" ], "entityId": "caues-mbp", "entityType": "host" } ``` - Fix a problem on the `Service` entity definition removing the `datastream.type` and moving it to `source_data_stream.type` due to ECS conflicts. - Moves some common field definitions to **observability-shared** plugin, and updated APM and Inventory plugin. --------- Co-authored-by: kibanamachine <[email protected]> (cherry picked from commit a91d007)
kibanamachine · Oct 4, 2024 · 668e902 · 668e902
1 parent bc43469
commit 668e902
Show file tree

Hide file tree

Showing 32 changed files with 250 additions and 83 deletions.
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts
@@ -20,7 +20,7 @@ const serviceTransactionFilter = (additionalFilters: string[] = []) => {
 
 export const builtInServicesFromLogsEntityDefinition: EntityDefinition =
   entityDefinitionSchema.parse({
-    version: '0.2.0',
+    version: '0.3.0',
     id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`,
     name: 'Services from ECS data',
     description:
@@ -46,8 +46,15 @@ export const builtInServicesFromLogsEntityDefinition: EntityDefinition =
     displayNameTemplate: '{{service.name}}',
     metadata: [
       { source: '_index', destination: 'sourceIndex' },
+      {
+        source: 'data_stream.type',
+        destination: 'source_data_stream.type',
+      },
+      {
+        source: 'data_stream.dataset',
+        destination: 'source_data_stream.dataset',
+      },
       { source: 'agent.name', aggregation: { type: 'terms', limit: 100 } },
-      'data_stream.type',
       'service.environment',
       'service.name',
       'service.namespace',

diff --git a/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts
@@ -31,6 +31,7 @@ export async function findEntityDefinitions({
   page = 1,
   perPage = 10,
   includeState = false,
+  type,
 }: {
   soClient: SavedObjectsClientContract;
   esClient: ElasticsearchClient;
@@ -39,12 +40,14 @@ export async function findEntityDefinitions({
   page?: number;
   perPage?: number;
   includeState?: boolean;
+  type?: string;
 }): Promise<EntityDefinition[] | EntityDefinitionWithState[]> {
   const filter = compact([
     typeof builtIn === 'boolean'
       ? `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${BUILT_IN_ID_PREFIX}*)`
       : undefined,
     id ? `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${id})` : undefined,
+    type ? `${SO_ENTITY_DEFINITION_TYPE}.attributes.type:(${type})` : undefined,
   ]).join(' AND ');
   const response = await soClient.find<EntityDefinition>({
     type: SO_ENTITY_DEFINITION_TYPE,

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -75,11 +75,15 @@ export class EntityClient {
     page = 1,
     perPage = 10,
     includeState = false,
+    type,
+    builtIn,
   }: {
     id?: string;
     page?: number;
     perPage?: number;
     includeState?: boolean;
+    type?: string;
+    builtIn?: boolean;
   }) {
     const definitions = await findEntityDefinitions({
       esClient: this.options.esClient,
@@ -88,6 +92,8 @@ export class EntityClient {
       perPage,
       id,
       includeState,
+      type,
+      builtIn,
     });
 
     return { definitions };

diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/entities.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/entities.ts
@@ -5,13 +5,8 @@
  * 2.0.
  */
 
-export const ENTITY = 'entity';
-export const LAST_SEEN = 'entity.lastSeenTimestamp';
-export const FIRST_SEEN = 'entity.firstSeenTimestamp';
-export const ENTITY_ID = 'entity.id';
 export const ENTITY_METRICS_LATENCY = 'entity.metrics.latency';
 export const ENTITY_METRICS_LOG_ERROR_RATE = 'entity.metrics.logErrorRate';
 export const ENTITY_METRICS_LOG_RATE = 'entity.metrics.logRate';
 export const ENTITY_METRICS_THROUGHPUT = 'entity.metrics.throughput';
 export const ENTITY_METRICS_FAILED_TRANSACTION_RATE = 'entity.metrics.failedTransactionRate';
-export const ENTITY_TYPE = 'entity.type';
diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts
@@ -5,7 +5,10 @@
  * 2.0.
  */
 import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { FIRST_SEEN, LAST_SEEN } from '../../../common/es_fields/entities';
+import {
+  ENTITY_FIRST_SEEN,
+  ENTITY_LAST_SEEN,
+} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch';
 import type { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
 import { getEntityLatestServices } from './get_entity_latest_services';
 import type { EntityLatestServiceRaw } from './types';
@@ -19,14 +22,14 @@ export function entitiesRangeQuery(start?: number, end?: number): QueryDslQueryC
   return [
     {
       range: {
-        [LAST_SEEN]: {
+        [ENTITY_LAST_SEEN]: {
           gte: start,
         },
       },
     },
     {
       range: {
-        [FIRST_SEEN]: {
+        [ENTITY_FIRST_SEEN]: {
           lte: end,
         },
       },

diff --git a/.../observability_solution/apm/server/routes/entities/get_entity_history_services_metrics.ts b/.../observability_solution/apm/server/routes/entities/get_entity_history_services_metrics.ts
@@ -5,16 +5,18 @@
  * 2.0.
  */
 
-import { termsQuery, rangeQuery } from '@kbn/observability-plugin/server';
-import { EntityMetrics } from '../../../common/entities/types';
+import { rangeQuery, termsQuery } from '@kbn/observability-plugin/server';
 import {
   ENTITY_ID,
+  ENTITY_LAST_SEEN,
+} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch';
+import { EntityMetrics } from '../../../common/entities/types';
+import {
   ENTITY_METRICS_FAILED_TRANSACTION_RATE,
   ENTITY_METRICS_LATENCY,
   ENTITY_METRICS_LOG_ERROR_RATE,
   ENTITY_METRICS_LOG_RATE,
   ENTITY_METRICS_THROUGHPUT,
-  LAST_SEEN,
 } from '../../../common/es_fields/entities';
 import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
 
@@ -39,7 +41,10 @@ export async function getEntityHistoryServicesMetrics({
       track_total_hits: false,
       query: {
         bool: {
-          filter: [...rangeQuery(start, end, LAST_SEEN), ...termsQuery(ENTITY_ID, ...entityIds)],
+          filter: [
+            ...rangeQuery(start, end, ENTITY_LAST_SEEN),
+            ...termsQuery(ENTITY_ID, ...entityIds),
+          ],
         },
       },
       aggs: {

diff --git a/...servability_solution/apm/server/routes/entities/get_entity_history_services_timeseries.ts b/...servability_solution/apm/server/routes/entities/get_entity_history_services_timeseries.ts
@@ -5,20 +5,20 @@
  * 2.0.
  */
 
-import { termsQuery, rangeQuery } from '@kbn/observability-plugin/server';
 import { getBucketSize } from '@kbn/apm-data-access-plugin/common';
+import { rangeQuery, termsQuery } from '@kbn/observability-plugin/server';
+import { ENTITY_LAST_SEEN } from '@kbn/observability-shared-plugin/common/field_names/elasticsearch';
 import { keyBy } from 'lodash';
+import { SERVICE_NAME } from '../../../common/es_fields/apm';
 import {
   ENTITY_METRICS_FAILED_TRANSACTION_RATE,
   ENTITY_METRICS_LATENCY,
   ENTITY_METRICS_LOG_ERROR_RATE,
   ENTITY_METRICS_LOG_RATE,
   ENTITY_METRICS_THROUGHPUT,
-  LAST_SEEN,
 } from '../../../common/es_fields/entities';
-import { SERVICE_NAME } from '../../../common/es_fields/apm';
-import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
 import { environmentQuery } from '../../../common/utils/environment_query';
+import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
 
 interface Params {
   entitiesESClient: EntitiesESClient;
@@ -48,7 +48,7 @@ export async function getEntityHistoryServicesTimeseries({
       query: {
         bool: {
           filter: [
-            ...rangeQuery(start, end, LAST_SEEN),
+            ...rangeQuery(start, end, ENTITY_LAST_SEEN),
             ...termsQuery(SERVICE_NAME, ...serviceNames),
             ...environmentQuery(environment),
           ],

diff --git a/...k/plugins/observability_solution/apm/server/routes/entities/get_entity_latest_services.ts b/...k/plugins/observability_solution/apm/server/routes/entities/get_entity_latest_services.ts
@@ -7,12 +7,11 @@
 
 import { kqlQuery, termQuery } from '@kbn/observability-plugin/server';
 import {
-  AGENT_NAME,
-  DATA_STEAM_TYPE,
-  SERVICE_ENVIRONMENT,
-  SERVICE_NAME,
-} from '../../../common/es_fields/apm';
-import { ENTITY, ENTITY_TYPE } from '../../../common/es_fields/entities';
+  ENTITY,
+  ENTITY_TYPE,
+  SOURCE_DATA_STREAM_TYPE,
+} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch';
+import { AGENT_NAME, SERVICE_ENVIRONMENT, SERVICE_NAME } from '../../../common/es_fields/apm';
 import { environmentQuery } from '../../../common/utils/environment_query';
 import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
 import { entitiesRangeQuery } from './get_entities';
@@ -40,7 +39,7 @@ export async function getEntityLatestServices({
       body: {
         size,
         track_total_hits: false,
-        _source: [AGENT_NAME, ENTITY, DATA_STEAM_TYPE, SERVICE_NAME, SERVICE_ENVIRONMENT],
+        _source: [AGENT_NAME, ENTITY, SOURCE_DATA_STREAM_TYPE, SERVICE_NAME, SERVICE_ENVIRONMENT],
         query: {
           bool: {
             filter: [

diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts
@@ -15,7 +15,7 @@ export interface EntityLatestServiceRaw {
   agent: {
     name: AgentName[];
   };
-  data_stream: {
+  source_data_stream: {
     type: string[];
   };
   service: {

diff --git a/...ck/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts b/...ck/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts
@@ -18,7 +18,7 @@ describe('mergeEntities', () => {
           environment: 'test',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: ['metrics', 'logs'] },
+        source_data_stream: { type: ['metrics', 'logs'] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -64,7 +64,7 @@ describe('mergeEntities', () => {
           environment: 'env-service-1',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: ['foo'] },
+        source_data_stream: { type: ['foo'] },
         entity: {
           firstSeenTimestamp: '2024-03-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-03-05T10:34:40.810Z',
@@ -85,7 +85,7 @@ describe('mergeEntities', () => {
           environment: 'env-service-2',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: ['bar'] },
+        source_data_stream: { type: ['bar'] },
         entity: {
           firstSeenTimestamp: '2024-03-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-03-05T10:34:40.810Z',
@@ -106,7 +106,7 @@ describe('mergeEntities', () => {
           environment: 'env-service-3',
         },
         agent: { name: ['java'] },
-        data_stream: { type: ['baz'] },
+        source_data_stream: { type: ['baz'] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -127,7 +127,7 @@ describe('mergeEntities', () => {
           environment: 'env-service-4',
         },
         agent: { name: ['java'] },
-        data_stream: { type: ['baz'] },
+        source_data_stream: { type: ['baz'] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -204,7 +204,7 @@ describe('mergeEntities', () => {
           environment: 'test',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: ['metrics', 'logs'] },
+        source_data_stream: { type: ['metrics', 'logs'] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -225,7 +225,7 @@ describe('mergeEntities', () => {
           environment: 'test',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: ['metrics', 'logs'] },
+        source_data_stream: { type: ['metrics', 'logs'] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -246,7 +246,7 @@ describe('mergeEntities', () => {
           environment: 'prod',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: ['foo'] },
+        source_data_stream: { type: ['foo'] },
         entity: {
           firstSeenTimestamp: '2024-23-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-23-05T10:34:40.810Z',
@@ -305,7 +305,7 @@ describe('mergeEntities', () => {
           environment: undefined,
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: [] },
+        source_data_stream: { type: [] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -348,7 +348,7 @@ describe('mergeEntities', () => {
           name: 'service-1',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: [] },
+        source_data_stream: { type: [] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -368,7 +368,7 @@ describe('mergeEntities', () => {
           name: 'service-1',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: [] },
+        source_data_stream: { type: [] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -420,7 +420,7 @@ describe('mergeEntities', () => {
           name: 'service-1',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: [] },
+        source_data_stream: { type: [] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -463,7 +463,7 @@ describe('mergeEntities', () => {
           name: 'service-1',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: [] },
+        source_data_stream: { type: [] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -483,7 +483,7 @@ describe('mergeEntities', () => {
           name: 'service-1',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: [] },
+        source_data_stream: { type: [] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
@@ -536,7 +536,7 @@ describe('mergeEntities', () => {
           environment: 'test',
         },
         agent: { name: ['nodejs'] },
-        data_stream: { type: ['metrics'] },
+        source_data_stream: { type: ['metrics'] },
         entity: {
           firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
           lastSeenTimestamp: '2024-06-05T10:34:40.810Z',

diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts
@@ -53,7 +53,7 @@ function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServic
   if (!existingEntity) {
     return {
       ...commonEntityFields,
-      dataStreamTypes: entity.data_stream.type,
+      dataStreamTypes: entity.source_data_stream.type,
       environments: compact([entity?.service.environment]),
       metrics: [entity.entity.metrics],
       hasLogMetrics,
@@ -62,7 +62,7 @@ function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServic
   return {
     ...commonEntityFields,
     dataStreamTypes: uniq(
-      compact([...(existingEntity?.dataStreamTypes ?? []), ...entity.data_stream.type])
+      compact([...(existingEntity?.dataStreamTypes ?? []), ...entity.source_data_stream.type])
     ),
     environments: uniq(compact([...existingEntity?.environments, entity?.service.environment])),
     metrics: [...existingEntity?.metrics, entity.entity.metrics],

diff --git a/x-pack/plugins/observability_solution/infra/kibana.jsonc b/x-pack/plugins/observability_solution/infra/kibana.jsonc
@@ -35,7 +35,8 @@
       "usageCollection",
       "visTypeTimeseries",
       "apmDataAccess",
-      "logsDataAccess"
+      "logsDataAccess",
+      "entityManager"
     ],
     "optionalPlugins": [
       "spaces",

diff --git a/x-pack/plugins/observability_solution/infra/server/infra_server.ts b/x-pack/plugins/observability_solution/infra/server/infra_server.ts
@@ -34,6 +34,7 @@ import { initProfilingRoutes } from './routes/profiling';
 import { initServicesRoute } from './routes/services';
 import { initCustomDashboardsRoutes } from './routes/custom_dashboards/custom_dashboards';
 import { InfraBackendLibs } from './lib/infra_types';
+import { initEntitiesConfigurationRoutes } from './routes/entities';
 
 export const registerRoutes = (libs: InfraBackendLibs) => {
   initIpToHostName(libs);
@@ -63,4 +64,5 @@ export const registerRoutes = (libs: InfraBackendLibs) => {
   initProfilingRoutes(libs);
   initServicesRoute(libs);
   initCustomDashboardsRoutes(libs.framework);
+  initEntitiesConfigurationRoutes(libs);
 };