From ac4eebca60c0d633271a6c7bc72560aa91d2ebbb Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Thu, 21 Nov 2024 18:52:37 +0100 Subject: [PATCH 1/4] Inventory k8s entities fixes --- .../kubernetes/common/ecs_index_patterns.ts | 2 +- .../built_in/kubernetes/ecs/cron_job.ts | 4 +-- .../built_in/kubernetes/ecs/daemon_set.ts | 2 +- .../built_in/kubernetes/ecs/deployment.ts | 4 +-- .../entities/built_in/kubernetes/ecs/job.ts | 4 +-- .../entities/built_in/kubernetes/ecs/node.ts | 4 +-- .../entities/built_in/kubernetes/ecs/pod.ts | 2 +- .../built_in/kubernetes/ecs/replica_set.ts | 3 +- .../built_in/kubernetes/ecs/stateful_set.ts | 4 +-- .../built_in/kubernetes/semconv/node.ts | 2 +- .../kubernetes/semconv/replica_set.ts | 2 +- .../generate_latest_processors.ts | 20 ++++++++--- .../generate_metadata_aggregations.ts | 20 ++++++++--- .../asset_details/hooks/use_entity_summary.ts | 10 ++++-- .../tabs/processes/processes.tsx | 4 +-- .../infra/server/routes/entities/index.ts | 6 ++-- .../public/components/entity_icon/index.tsx | 2 +- .../hooks/use_detail_view_redirect.test.ts | 35 +++++++++++++------ .../public/hooks/use_detail_view_redirect.ts | 23 ++++++------ .../common/entity/entity_types.ts | 10 +++--- .../common/entity/index.ts | 2 +- .../observability_shared/common/index.ts | 2 +- 22 files changed, 108 insertions(+), 59 deletions(-) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts index 14e2766cac2b2..63536fdfcea57 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts @@ -5,4 +5,4 @@ * 2.0. */ -export const commonEcsIndexPatterns = ['metrics-kubernetes*', 'logs-*']; +export const commonEcsIndexPatterns = ['metrics-kubernetes*', 'metrics-apm*', 'logs-*']; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cron_job.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cron_job.ts index 7849dcdc73f5b..06bc9dba9fce7 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cron_job.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cron_job.ts @@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata'; export const builtInKubernetesCronJobEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}kubernetes_cron_job_ecs`, - filter: 'kubernetes.cronjob.uid : *', + filter: 'kubernetes.cronjob.name : *', managed: true, version: '0.1.0', name: 'Kubernetes CronJob from ECS data', @@ -21,7 +21,7 @@ export const builtInKubernetesCronJobEcsEntityDefinition: EntityDefinition = 'This definition extracts Kubernetes cron job entities from the Kubernetes integration data streams', type: 'k8s.cronjob.ecs', indexPatterns: commonEcsIndexPatterns, - identityFields: ['kubernetes.cronjob.uid'], + identityFields: ['kubernetes.cronjob.name'], displayNameTemplate: '{{kubernetes.cronjob.name}}', latest: { timestampField: '@timestamp', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/daemon_set.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/daemon_set.ts index 5b57cdd6ae2f8..c69a1c5c7e625 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/daemon_set.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/daemon_set.ts @@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata'; export const builtInKubernetesDaemonSetEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}kubernetes_daemon_set_ecs`, - filter: 'kubernetes.daemonset.uid : *', + filter: 'kubernetes.daemonset.name : *', managed: true, version: '0.1.0', name: 'Kubernetes DaemonSet from ECS data', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/deployment.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/deployment.ts index d33c14db7e2c9..f8e8f920e2f47 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/deployment.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/deployment.ts @@ -13,7 +13,7 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns'; export const builtInKubernetesDeploymentEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}kubernetes_deployment_ecs`, - filter: 'kubernetes.deployment.uid : *', + filter: 'kubernetes.deployment.name : *', managed: true, version: '0.1.0', name: 'Kubernetes Deployment from ECS data', @@ -21,7 +21,7 @@ export const builtInKubernetesDeploymentEcsEntityDefinition: EntityDefinition = 'This definition extracts Kubernetes deployment entities from the Kubernetes integration data streams', type: 'k8s.deployment.ecs', indexPatterns: commonEcsIndexPatterns, - identityFields: ['kubernetes.deployment.uid'], + identityFields: ['kubernetes.deployment.name'], displayNameTemplate: '{{kubernetes.deployment.name}}', latest: { timestampField: '@timestamp', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/job.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/job.ts index 92c6d13251553..4efc41dc9ea81 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/job.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/job.ts @@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata'; export const builtInKubernetesJobEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}kubernetes_job_ecs`, - filter: 'kubernetes.job.uid : *', + filter: 'kubernetes.job.name : *', managed: true, version: '0.1.0', name: 'Kubernetes Job from ECS data', @@ -21,7 +21,7 @@ export const builtInKubernetesJobEcsEntityDefinition: EntityDefinition = 'This definition extracts Kubernetes job entities from the Kubernetes integration data streams', type: 'k8s.job.ecs', indexPatterns: commonEcsIndexPatterns, - identityFields: ['kubernetes.job.uid'], + identityFields: ['kubernetes.job.name'], displayNameTemplate: '{{kubernetes.job.name}}', latest: { timestampField: '@timestamp', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/node.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/node.ts index f3fdcdfaf04b4..033bd8313928d 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/node.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/node.ts @@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata'; export const builtInKubernetesNodeEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}kubernetes_node_ecs`, - filer: 'kubernetes.node.uid : *', + filer: 'kubernetes.node.name : *', managed: true, version: '0.1.0', name: 'Kubernetes Node from ECS data', @@ -21,7 +21,7 @@ export const builtInKubernetesNodeEcsEntityDefinition: EntityDefinition = 'This definition extracts Kubernetes node entities from the Kubernetes integration data streams', type: 'k8s.node.ecs', indexPatterns: commonEcsIndexPatterns, - identityFields: ['kubernetes.node.uid'], + identityFields: ['kubernetes.node.name'], displayNameTemplate: '{{kubernetes.node.name}}', latest: { timestampField: '@timestamp', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts index 7aa53da6e5a7d..9742f8be00dda 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts @@ -21,7 +21,7 @@ export const builtInKubernetesPodEcsEntityDefinition: EntityDefinition = 'This definition extracts Kubernetes pod entities from the Kubernetes integration data streams', type: 'k8s.pod.ecs', indexPatterns: commonEcsIndexPatterns, - identityFields: ['kubernetes.pod.name'], + identityFields: ['kubernetes.pod.uid'], displayNameTemplate: '{{kubernetes.pod.name}}', latest: { timestampField: '@timestamp', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/replica_set.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/replica_set.ts index cc059c14979d0..e9f534be8f1db 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/replica_set.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/replica_set.ts @@ -13,6 +13,7 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns'; export const builtInKubernetesReplicaSetEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}kubernetes_replica_set_ecs`, + filer: 'kubernetes.replicaset.name : *', managed: true, version: '0.1.0', name: 'Kubernetes ReplicaSet from ECS data', @@ -20,7 +21,7 @@ export const builtInKubernetesReplicaSetEcsEntityDefinition: EntityDefinition = 'This definition extracts Kubernetes replica set entities from the Kubernetes integration data streams', type: 'k8s.replicaset.ecs', indexPatterns: commonEcsIndexPatterns, - identityFields: ['kubernetes.replicaset.uid'], + identityFields: ['kubernetes.replicaset.name'], displayNameTemplate: '{{kubernetes.replicaset.name}}', latest: { timestampField: '@timestamp', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/stateful_set.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/stateful_set.ts index 79f9d4489216f..927c8a259276d 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/stateful_set.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/stateful_set.ts @@ -13,7 +13,7 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns'; export const builtInKubernetesStatefulSetEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}kubernetes_stateful_set_ecs`, - filter: 'kubernetes.statefulset.uid : *', + filter: 'kubernetes.statefulset.name : *', managed: true, version: '0.1.0', name: 'Kubernetes StatefulSet from ECS data', @@ -21,7 +21,7 @@ export const builtInKubernetesStatefulSetEcsEntityDefinition: EntityDefinition = 'This definition extracts Kubernetes stateful set entities from the Kubernetes integration data streams', type: 'k8s.statefulset.ecs', indexPatterns: commonEcsIndexPatterns, - identityFields: ['kubernetes.statefulset.uid'], + identityFields: ['kubernetes.statefulset.name'], displayNameTemplate: '{{kubernetes.statefulset.name}}', latest: { timestampField: '@timestamp', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/node.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/node.ts index 456f030421075..4b70ea5853fbf 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/node.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/node.ts @@ -22,7 +22,7 @@ export const builtInKubernetesNodeSemConvEntityDefinition: EntityDefinition = type: 'k8s.node.otel', indexPatterns: commonOtelIndexPatterns, identityFields: ['k8s.node.uid'], - displayNameTemplate: '{{k8s.node.uid}}', + displayNameTemplate: '{{k8s.node.name}}', latest: { timestampField: '@timestamp', lookbackPeriod: '10m', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/replica_set.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/replica_set.ts index 47bad6bf8a641..7759ce499f920 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/replica_set.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/replica_set.ts @@ -19,7 +19,7 @@ export const builtInKubernetesReplicaSetSemConvEntityDefinition: EntityDefinitio name: 'Kubernetes ReplicaSet from SemConv data', description: 'This definition extracts Kubernetes replica set entities using data collected with OpenTelemetry', - type: 'kubernetes_replica_set_semconv', + type: 'k8s.replicaset.otel', indexPatterns: commonOtelIndexPatterns, identityFields: ['k8s.replicaset.name'], displayNameTemplate: '{{k8s.replicaset.name}}', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index 335c135448b10..633866ddd0008 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -30,11 +30,23 @@ function mapDestinationToPainless(metadata: MetadataField) { } function createMetadataPainlessScript(definition: EntityDefinition) { - if (!definition.metadata) { - return ''; - } + const displayNameField = definition.displayNameTemplate.replace(/[{}]/g, ''); + const shouldAddDisplayNameMetadata = definition.identityFields.some( + ({ field }) => field !== displayNameField + ); + const displayNameMetadata: EntityDefinition['metadata'] = shouldAddDisplayNameMetadata + ? [ + { + source: displayNameField, + destination: displayNameField, + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ] + : undefined; + + const combinedMetadata = [...(definition.metadata ?? []), ...(displayNameMetadata ?? [])]; - return definition.metadata.reduce((acc, metadata) => { + return combinedMetadata.reduce((acc, metadata) => { const { destination, source } = metadata; const optionalFieldPath = destination.replaceAll('.', '?.'); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 796d1e25b55ec..81aa29678e592 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -8,11 +8,23 @@ import { EntityDefinition } from '@kbn/entities-schema'; export function generateLatestMetadataAggregations(definition: EntityDefinition) { - if (!definition.metadata) { - return {}; - } + const displayNameField = definition.displayNameTemplate.replace(/[{}]/g, ''); + const shouldAddDisplayNameMetadata = definition.identityFields.some( + ({ field }) => field !== displayNameField + ); + const displayNameMetadata: EntityDefinition['metadata'] = shouldAddDisplayNameMetadata + ? [ + { + source: displayNameField, + destination: displayNameField, + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ] + : undefined; + + const combinedMetadata = [...(definition.metadata ?? []), ...(displayNameMetadata ?? [])]; - return definition.metadata.reduce((aggs, metadata) => { + return combinedMetadata.reduce((aggs, metadata) => { const lookbackPeriod = metadata.aggregation.lookbackPeriod || definition.latest.lookbackPeriod; let agg; if (metadata.aggregation.type === 'terms') { diff --git a/x-pack/plugins/observability_solution/infra/public/components/asset_details/hooks/use_entity_summary.ts b/x-pack/plugins/observability_solution/infra/public/components/asset_details/hooks/use_entity_summary.ts index e62defaac6d4b..500dd02c63e18 100644 --- a/x-pack/plugins/observability_solution/infra/public/components/asset_details/hooks/use_entity_summary.ts +++ b/x-pack/plugins/observability_solution/infra/public/components/asset_details/hooks/use_entity_summary.ts @@ -6,10 +6,16 @@ */ import * as z from '@kbn/zod'; -import { EntityDataStreamType, ENTITY_TYPES } from '@kbn/observability-shared-plugin/common'; +import { + EntityDataStreamType, + BUILT_IN_ENTITY_TYPES, +} from '@kbn/observability-shared-plugin/common'; import { useFetcher } from '../../../hooks/use_fetcher'; -const EntityTypeSchema = z.union([z.literal(ENTITY_TYPES.HOST), z.literal(ENTITY_TYPES.CONTAINER)]); +const EntityTypeSchema = z.union([ + z.literal(BUILT_IN_ENTITY_TYPES.HOST), + z.literal(BUILT_IN_ENTITY_TYPES.CONTAINER), +]); const EntityDataStreamSchema = z.union([ z.literal(EntityDataStreamType.METRICS), z.literal(EntityDataStreamType.LOGS), diff --git a/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/processes/processes.tsx b/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/processes/processes.tsx index 2177cd0509085..57a07d4dc296e 100644 --- a/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/processes/processes.tsx +++ b/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/processes/processes.tsx @@ -23,7 +23,7 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { getFieldByType } from '@kbn/metrics-data-access-plugin/common'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import useLocalStorage from 'react-use/lib/useLocalStorage'; -import { ENTITY_TYPES } from '@kbn/observability-shared-plugin/common'; +import { BUILT_IN_ENTITY_TYPES } from '@kbn/observability-shared-plugin/common'; import { useSourceContext } from '../../../../containers/metrics_source'; import { isPending, useFetcher } from '../../../../hooks/use_fetcher'; import { parseSearchString } from './parse_search_string'; @@ -58,7 +58,7 @@ export const Processes = () => { const { request$ } = useRequestObservable(); const { isActiveTab } = useTabSwitcherContext(); const { dataStreams, status: dataStreamsStatus } = useEntitySummary({ - entityType: ENTITY_TYPES.HOST, + entityType: BUILT_IN_ENTITY_TYPES.HOST, entityId: asset.name, }); const addMetricsCalloutId: AddMetricsCalloutKey = 'hostProcesses'; diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts index 30be4fc9da498..b551f8f83b4dd 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts @@ -9,7 +9,7 @@ import { schema } from '@kbn/config-schema'; import { METRICS_APP_ID } from '@kbn/deeplinks-observability/constants'; import { entityCentricExperience } from '@kbn/observability-plugin/common'; import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { ENTITY_TYPES } from '@kbn/observability-shared-plugin/common'; +import { BUILT_IN_ENTITY_TYPES } from '@kbn/observability-shared-plugin/common'; import { getInfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client'; import { InfraBackendLibs } from '../../lib/infra_types'; import { getDataStreamTypes } from './get_data_stream_types'; @@ -24,8 +24,8 @@ export const initEntitiesConfigurationRoutes = (libs: InfraBackendLibs) => { validate: { params: schema.object({ entityType: schema.oneOf([ - schema.literal(ENTITY_TYPES.HOST), - schema.literal(ENTITY_TYPES.CONTAINER), + schema.literal(BUILT_IN_ENTITY_TYPES.HOST), + schema.literal(BUILT_IN_ENTITY_TYPES.CONTAINER), ]), entityId: schema.string(), }), diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entity_icon/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entity_icon/index.tsx index 4da8fd3103c41..239d441b5d4e6 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/entity_icon/index.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/entity_icon/index.tsx @@ -45,7 +45,7 @@ export function EntityIcon({ entity }: EntityIconProps) { return ; } - if (entity.entityType.startsWith('kubernetes')) { + if (entity.entityType.startsWith('k8s')) { return ; } diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.test.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.test.ts index 233c1a1076b79..403e09ab22916 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.test.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.test.ts @@ -10,7 +10,7 @@ import { useDetailViewRedirect } from './use_detail_view_redirect'; import { useKibana } from './use_kibana'; import { CONTAINER_ID, - ENTITY_TYPES, + BUILT_IN_ENTITY_TYPES, HOST_NAME, SERVICE_NAME, } from '@kbn/observability-shared-plugin/common'; @@ -134,15 +134,30 @@ describe('useDetailViewRedirect', () => { }); [ - [ENTITY_TYPES.KUBERNETES.CLUSTER.ecs, 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c'], - [ENTITY_TYPES.KUBERNETES.CLUSTER.semconv, 'kubernetes_otel-cluster-overview'], - [ENTITY_TYPES.KUBERNETES.CRONJOB.ecs, 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013'], - [ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs, 'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013'], - [ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs, 'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013'], - [ENTITY_TYPES.KUBERNETES.JOB.ecs, 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013'], - [ENTITY_TYPES.KUBERNETES.NODE.ecs, 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013'], - [ENTITY_TYPES.KUBERNETES.POD.ecs, 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013'], - [ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs, 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013'], + [ + BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.ecs, + 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c', + ], + [BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.semconv, 'kubernetes_otel-cluster-overview'], + [ + BUILT_IN_ENTITY_TYPES.KUBERNETES.CRONJOB.ecs, + 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013', + ], + [ + BUILT_IN_ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs, + 'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013', + ], + [ + BUILT_IN_ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs, + 'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013', + ], + [BUILT_IN_ENTITY_TYPES.KUBERNETES.JOB.ecs, 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013'], + [BUILT_IN_ENTITY_TYPES.KUBERNETES.NODE.ecs, 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013'], + [BUILT_IN_ENTITY_TYPES.KUBERNETES.POD.ecs, 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013'], + [ + BUILT_IN_ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs, + 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013', + ], ].forEach(([entityType, dashboardId]) => { it(`getEntityRedirectUrl should return the correct URL for ${entityType} entity`, () => { const entity: InventoryEntity = { diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts index 4df4fa4ca1f96..b7f56c65022f4 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts @@ -6,7 +6,7 @@ */ import { ASSET_DETAILS_LOCATOR_ID, - ENTITY_TYPES, + BUILT_IN_ENTITY_TYPES, SERVICE_OVERVIEW_LOCATOR_ID, type AssetDetailsLocatorParams, type ServiceOverviewParams, @@ -20,15 +20,18 @@ import type { InventoryEntity } from '../../common/entities'; import { useKibana } from './use_kibana'; const KUBERNETES_DASHBOARDS_IDS: Record = { - [ENTITY_TYPES.KUBERNETES.CLUSTER.ecs]: 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c', - [ENTITY_TYPES.KUBERNETES.CLUSTER.semconv]: 'kubernetes_otel-cluster-overview', - [ENTITY_TYPES.KUBERNETES.CRONJOB.ecs]: 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013', - [ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs]: 'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013', - [ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs]: 'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013', - [ENTITY_TYPES.KUBERNETES.JOB.ecs]: 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013', - [ENTITY_TYPES.KUBERNETES.NODE.ecs]: 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013', - [ENTITY_TYPES.KUBERNETES.POD.ecs]: 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013', - [ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs]: 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.ecs]: 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.semconv]: 'kubernetes_otel-cluster-overview', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.CRONJOB.ecs]: 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs]: + 'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs]: + 'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.JOB.ecs]: 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.NODE.ecs]: 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.POD.ecs]: 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs]: + 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013', }; export const useDetailViewRedirect = () => { diff --git a/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts b/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts index 4d8be9efc59c6..167f7b56544ca 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts @@ -6,11 +6,11 @@ */ const createKubernetesEntity = (base: T) => ({ - ecs: `kubernetes_${base}_ecs` as const, - semconv: `kubernetes_${base}_semconv` as const, + ecs: `k8s.${base}.ecs` as const, + semconv: `k8s.${base}.semconv` as const, }); -export const ENTITY_TYPES = { +export const BUILT_IN_ENTITY_TYPES = { HOST: 'host', CONTAINER: 'container', SERVICE: 'service', @@ -18,12 +18,12 @@ export const ENTITY_TYPES = { CLUSTER: createKubernetesEntity('cluster'), CONTAINER: createKubernetesEntity('container'), CRONJOB: createKubernetesEntity('cron_job'), - DAEMONSET: createKubernetesEntity('daemon_set'), + DAEMONSET: createKubernetesEntity('daemonset'), DEPLOYMENT: createKubernetesEntity('deployment'), JOB: createKubernetesEntity('job'), NAMESPACE: createKubernetesEntity('namespace'), NODE: createKubernetesEntity('node'), POD: createKubernetesEntity('pod'), - STATEFULSET: createKubernetesEntity('stateful_set'), + STATEFULSET: createKubernetesEntity('statefulset'), }, } as const; diff --git a/x-pack/plugins/observability_solution/observability_shared/common/entity/index.ts b/x-pack/plugins/observability_solution/observability_shared/common/entity/index.ts index adc07a2931b60..92ae5701f8000 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/entity/index.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/entity/index.ts @@ -5,5 +5,5 @@ * 2.0. */ -export { ENTITY_TYPES } from './entity_types'; +export { BUILT_IN_ENTITY_TYPES } from './entity_types'; export { EntityDataStreamType } from './entity_data_stream_types'; diff --git a/x-pack/plugins/observability_solution/observability_shared/common/index.ts b/x-pack/plugins/observability_solution/observability_shared/common/index.ts index f483bcc5dc269..a8e26366ab4b3 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/index.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/index.ts @@ -219,4 +219,4 @@ export { export { COMMON_OBSERVABILITY_GROUPING } from './embeddable_grouping'; -export { ENTITY_TYPES, EntityDataStreamType } from './entity'; +export { BUILT_IN_ENTITY_TYPES, EntityDataStreamType } from './entity'; From 4c628dd2ec9474aa52f871cf8106813e9249122e Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Thu, 21 Nov 2024 20:46:53 +0100 Subject: [PATCH 2/4] Update synthtrace k8s entities --- .../src/lib/entities/index.ts | 4 +- .../lib/entities/kubernetes/cluster_entity.ts | 2 + .../entities/kubernetes/container_entity.ts | 2 + .../entities/kubernetes/cron_job_entity.ts | 7 ++-- .../entities/kubernetes/daemon_set_entity.ts | 6 ++- .../entities/kubernetes/deployment_entity.ts | 3 +- .../src/lib/entities/kubernetes/index.ts | 37 +++++++++++-------- .../{job_set_entity.ts => job_entity.ts} | 5 ++- .../lib/entities/kubernetes/node_entity.ts | 3 +- .../src/lib/entities/kubernetes/pod_entity.ts | 2 + .../lib/entities/kubernetes/replica_set.ts | 7 ++-- .../lib/entities/kubernetes/stateful_set.ts | 7 ++-- .../entities/entities_synthtrace_es_client.ts | 10 ++--- .../src/scenarios/k8s_entities.ts | 2 +- .../src/scenarios/many_entities.ts | 2 +- 15 files changed, 58 insertions(+), 41 deletions(-) rename packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/{job_set_entity.ts => job_entity.ts} (91%) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts index 8665ef0120e25..6cb948fe35091 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts @@ -15,7 +15,7 @@ import { k8sClusterJobEntity } from './kubernetes/cluster_entity'; import { k8sCronJobEntity } from './kubernetes/cron_job_entity'; import { k8sDaemonSetEntity } from './kubernetes/daemon_set_entity'; import { k8sDeploymentEntity } from './kubernetes/deployment_entity'; -import { k8sJobSetEntity } from './kubernetes/job_set_entity'; +import { k8sJobEntity } from './kubernetes/job_entity'; import { k8sNodeEntity } from './kubernetes/node_entity'; import { k8sPodEntity } from './kubernetes/pod_entity'; import { k8sReplicaSetEntity } from './kubernetes/replica_set'; @@ -52,7 +52,7 @@ export const entities = { k8sCronJobEntity, k8sDaemonSetEntity, k8sDeploymentEntity, - k8sJobSetEntity, + k8sJobEntity, k8sNodeEntity, k8sPodEntity, k8sReplicaSetEntity, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cluster_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cluster_entity.ts index 9fa4c81d86ffb..487ddc89a8c6f 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cluster_entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cluster_entity.ts @@ -23,6 +23,7 @@ export function k8sClusterJobEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { + 'entity.definition_id': 'cluster', 'entity.type': 'cluster', 'orchestrator.cluster.name': name, 'entity.id': entityId, @@ -31,6 +32,7 @@ export function k8sClusterJobEntity({ } return new K8sEntity(schema, { + 'entity.definition_id': 'cluster', 'entity.type': 'cluster', 'k8s.cluster.uid': name, 'entity.id': entityId, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/container_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/container_entity.ts index b05d412b0dd5c..116563e731f0c 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/container_entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/container_entity.ts @@ -23,6 +23,7 @@ export function k8sContainerEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { + 'entity.definition_id': 'container', 'entity.type': 'container', 'kubernetes.container.id': id, 'entity.id': entityId, @@ -31,6 +32,7 @@ export function k8sContainerEntity({ } return new K8sEntity(schema, { + 'entity.definition_id': 'container', 'entity.type': 'container', 'container.id': id, 'entity.id': entityId, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cron_job_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cron_job_entity.ts index 8590378e699fb..86e17b5d4cfc5 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cron_job_entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/cron_job_entity.ts @@ -27,9 +27,9 @@ export function k8sCronJobEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { - 'entity.type': 'cron_job', + 'entity.definition_id': 'cron_job', + 'entity.type': 'cronjob', 'kubernetes.cronjob.name': name, - 'kubernetes.cronjob.uid': uid, 'kubernetes.namespace': clusterName, 'entity.id': entityId, ...others, @@ -37,7 +37,8 @@ export function k8sCronJobEntity({ } return new K8sEntity(schema, { - 'entity.type': 'cron_job', + 'entity.definition_id': 'cron_job', + 'entity.type': 'cronjob', 'k8s.cronjob.name': name, 'k8s.cronjob.uid': uid, 'k8s.cluster.name': clusterName, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/daemon_set_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/daemon_set_entity.ts index 7e20b1c6d506f..59fe25dedf5a5 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/daemon_set_entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/daemon_set_entity.ts @@ -27,7 +27,8 @@ export function k8sDaemonSetEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { - 'entity.type': 'daemon_set', + 'entity.definition_id': 'daemon_set', + 'entity.type': 'daemonset', 'kubernetes.daemonset.name': name, 'kubernetes.daemonset.uid': uid, 'kubernetes.namespace': clusterName, @@ -37,7 +38,8 @@ export function k8sDaemonSetEntity({ } return new K8sEntity(schema, { - 'entity.type': 'daemon_set', + 'entity.definition_id': 'daemon_set', + 'entity.type': 'daemonset', 'k8s.daemonset.name': name, 'k8s.daemonset.uid': uid, 'k8s.cluster.name': clusterName, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/deployment_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/deployment_entity.ts index 7eabdd0827325..bc266d554f6df 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/deployment_entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/deployment_entity.ts @@ -27,9 +27,9 @@ export function k8sDeploymentEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { + 'entity.definition_id': 'deployment', 'entity.type': 'deployment', 'kubernetes.deployment.name': name, - 'kubernetes.deployment.uid': uid, 'kubernetes.namespace': clusterName, 'entity.id': entityId, ...others, @@ -37,6 +37,7 @@ export function k8sDeploymentEntity({ } return new K8sEntity(schema, { + 'entity.definition_id': 'deployment', 'entity.type': 'deployment', 'k8s.deployment.name': name, 'k8s.deployment.uid': uid, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts index 6da1decaab9ab..a6ed31a4e0a91 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts @@ -12,27 +12,27 @@ import { Serializable } from '../../serializable'; const identityFieldsMap: Record> = { ecs: { - pod: ['kubernetes.pod.name'], + pod: ['kubernetes.pod.uid'], cluster: ['orchestrator.cluster.name'], - cron_job: ['kubernetes.cronjob.name'], - daemon_set: ['kubernetes.daemonset.name'], + cronjob: ['kubernetes.cronjob.name'], + daemonset: ['kubernetes.daemonset.name'], deployment: ['kubernetes.deployment.name'], job: ['kubernetes.job.name'], node: ['kubernetes.node.name'], - replica_set: ['kubernetes.replicaset.name'], - stateful_set: ['kubernetes.statefulset.name'], + replicaset: ['kubernetes.replicaset.name'], + statefulset: ['kubernetes.statefulset.name'], container: ['kubernetes.container.id'], }, semconv: { - pod: ['k8s.pod.name'], + pod: ['k8s.pod.uid'], cluster: ['k8s.cluster.uid'], - cron_job: ['k8s.cronjob.name'], - daemon_set: ['k8s.daemonset.name'], - deployment: ['k8s.deployment.name'], - job: ['k8s.job.name'], + cronjob: ['k8s.cronjob.uid'], + daemonset: ['k8s.daemonset.uid'], + deployment: ['k8s.deployment.uid'], + job: ['k8s.job.uid'], node: ['k8s.node.uid'], - replica_set: ['k8s.replicaset.name'], - stateful_set: ['k8s.statefulset.name'], + replicaset: ['k8s.replicaset.uid'], + statefulset: ['k8s.statefulset.uid'], container: ['container.id'], }, }; @@ -41,10 +41,15 @@ export class K8sEntity extends Serializable { constructor(schema: Schema, fields: EntityFields) { const entityType = fields['entity.type']; if (entityType === undefined) { - throw new Error(`Entity type not defined: ${entityType}`); + throw new Error(`Entity type not defined`); } - const entityTypeWithSchema = `kubernetes_${entityType}_${schema}`; + const entityDefinitionId = fields['entity.definition_id']; + if (entityDefinitionId === undefined) { + throw new Error(`Entity definition id not defined`); + } + + const entityDefinitionWithSchema = `kubernetes_${entityDefinitionId}_${schema}`; const identityFields = identityFieldsMap[schema][entityType]; if (identityFields === undefined || identityFields.length === 0) { throw new Error( @@ -54,8 +59,8 @@ export class K8sEntity extends Serializable { super({ ...fields, - 'entity.type': entityTypeWithSchema, - 'entity.definition_id': `builtin_${entityTypeWithSchema}`, + 'entity.type': `k8s.${entityType}.${schema}`, + 'entity.definition_id': `builtin_${entityDefinitionWithSchema}`, 'entity.identity_fields': identityFields, 'entity.display_name': getDisplayName({ identityFields, fields }), 'entity.definition_version': '1.0.0', diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/job_set_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/job_entity.ts similarity index 91% rename from packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/job_set_entity.ts rename to packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/job_entity.ts index e0383563c7266..007f74d8a5bba 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/job_set_entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/job_entity.ts @@ -10,7 +10,7 @@ import { Schema } from '..'; import { K8sEntity } from '.'; -export function k8sJobSetEntity({ +export function k8sJobEntity({ schema, name, uid, @@ -27,9 +27,9 @@ export function k8sJobSetEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { + 'entity.definition_id': 'job', 'entity.type': 'job', 'kubernetes.job.name': name, - 'kubernetes.job.uid': uid, 'kubernetes.namespace': clusterName, 'entity.id': entityId, ...others, @@ -37,6 +37,7 @@ export function k8sJobSetEntity({ } return new K8sEntity(schema, { + 'entity.definition_id': 'job', 'entity.type': 'job', 'k8s.job.name': name, 'k8s.job.uid': uid, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/node_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/node_entity.ts index 283df5250d41d..4ab3441d7b82b 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/node_entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/node_entity.ts @@ -27,9 +27,9 @@ export function k8sNodeEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { + 'entity.definition_id': 'node', 'entity.type': 'node', 'kubernetes.node.name': name, - 'kubernetes.node.uid': uid, 'kubernetes.namespace': clusterName, 'entity.id': entityId, ...others, @@ -37,6 +37,7 @@ export function k8sNodeEntity({ } return new K8sEntity(schema, { + 'entity.definition_id': 'node', 'entity.type': 'node', 'k8s.node.uid': uid, 'k8s.cluster.name': clusterName, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/pod_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/pod_entity.ts index 1b71c4e39a4fc..47c24b01144f1 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/pod_entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/pod_entity.ts @@ -27,6 +27,7 @@ export function k8sPodEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { + 'entity.definition_id': 'pod', 'entity.type': 'pod', 'kubernetes.pod.name': name, 'kubernetes.pod.uid': uid, @@ -37,6 +38,7 @@ export function k8sPodEntity({ } return new K8sEntity(schema, { + 'entity.definition_id': 'pod', 'entity.type': 'pod', 'k8s.pod.name': name, 'k8s.pod.uid': uid, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/replica_set.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/replica_set.ts index fcf20c0530c30..b98397d2bee9b 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/replica_set.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/replica_set.ts @@ -27,9 +27,9 @@ export function k8sReplicaSetEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { - 'entity.type': 'replica_set', + 'entity.definition_id': 'replica_set', + 'entity.type': 'replicaset', 'kubernetes.replicaset.name': name, - 'kubernetes.replicaset.uid': uid, 'kubernetes.namespace': clusterName, 'entity.id': entityId, ...others, @@ -37,7 +37,8 @@ export function k8sReplicaSetEntity({ } return new K8sEntity(schema, { - 'entity.type': 'replica_set', + 'entity.definition_id': 'replica_set', + 'entity.type': 'replicaset', 'k8s.replicaset.name': name, 'k8s.replicaset.uid': uid, 'k8s.cluster.name': clusterName, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/stateful_set.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/stateful_set.ts index 58c603704ebc0..1857471c15635 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/stateful_set.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/stateful_set.ts @@ -27,9 +27,9 @@ export function k8sStatefulSetEntity({ }) { if (schema === 'ecs') { return new K8sEntity(schema, { - 'entity.type': 'stateful_set', + 'entity.definition_id': 'stateful_set', + 'entity.type': 'statefulset', 'kubernetes.statefulset.name': name, - 'kubernetes.statefulset.uid': uid, 'kubernetes.namespace': clusterName, 'entity.id': entityId, ...others, @@ -37,7 +37,8 @@ export function k8sStatefulSetEntity({ } return new K8sEntity(schema, { - 'entity.type': 'stateful_set', + 'entity.definition_id': 'stateful_set', + 'entity.type': 'statefulset', 'k8s.statefulset.name': name, 'k8s.statefulset.uid': uid, 'k8s.cluster.name': clusterName, diff --git a/packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts index 4c5d17111fca6..624b44eab0887 100644 --- a/packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts @@ -75,15 +75,13 @@ function getRoutingTransform() { return new Transform({ objectMode: true, transform(document: ESDocumentWithOperation, encoding, callback) { - const entityType: string | undefined = document['entity.type']; - if (entityType === undefined) { - throw new Error(`entity.type was not defined: ${JSON.stringify(document)}`); + const definitionId: string | undefined = document['entity.definition_id']; + if (definitionId === undefined) { + throw new Error(`entity.definition_id was not defined: ${JSON.stringify(document)}`); } - const entityIndexName = `${entityType}s`; document._action = { index: { - _index: - `.entities.v1.latest.builtin_${entityIndexName}_from_ecs_data`.toLocaleLowerCase(), + _index: `.entities.v1.latest.${definitionId}`.toLocaleLowerCase(), _id: document['entity.id'], }, }; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts b/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts index 7d94cc3180a7e..00d603e17abb8 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts @@ -99,7 +99,7 @@ const scenario: Scenario> = async (runOptions) => { }) .timestamp(timestamp), entities.k8s - .k8sJobSetEntity({ + .k8sJobEntity({ clusterName: CLUSTER_NAME, name: 'job_set_foo', schema, diff --git a/packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts b/packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts index 8b0d2afa5a971..19817d720a2a5 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts @@ -100,7 +100,7 @@ const scenario: Scenario> = async (runOptions) => { }) .timestamp(timestamp), entities.k8s - .k8sJobSetEntity({ + .k8sJobEntity({ clusterName: CLUSTER_NAME, name: 'job_set_foo', schema, From 790058906463f49d3b30ddbd350996099483d6eb Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Fri, 22 Nov 2024 11:13:01 +0100 Subject: [PATCH 3/4] ecs k8s service fix --- .../src/lib/entities/index.ts | 2 + .../src/lib/entities/kubernetes/index.ts | 1 + .../src/lib/entities/kubernetes/service.ts | 39 +++++++++++++++++++ .../src/scenarios/k8s_entities.ts | 19 ++++++++- .../kubernetes/common/ecs_index_patterns.ts | 2 +- .../public/hooks/use_detail_view_redirect.ts | 1 + .../common/entity/entity_types.ts | 1 + 7 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/service.ts diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts index 6cb948fe35091..1a2dc7536e366 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts @@ -20,6 +20,7 @@ import { k8sNodeEntity } from './kubernetes/node_entity'; import { k8sPodEntity } from './kubernetes/pod_entity'; import { k8sReplicaSetEntity } from './kubernetes/replica_set'; import { k8sStatefulSetEntity } from './kubernetes/stateful_set'; +import { k8sServiceEntity } from './kubernetes/service'; import { k8sContainerEntity } from './kubernetes/container_entity'; export type EntityDataStreamType = 'metrics' | 'logs' | 'traces'; @@ -57,6 +58,7 @@ export const entities = { k8sPodEntity, k8sReplicaSetEntity, k8sStatefulSetEntity, + k8sServiceEntity, k8sContainerEntity, }, }; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts index a6ed31a4e0a91..19306658341d6 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts @@ -21,6 +21,7 @@ const identityFieldsMap: Record> = { node: ['kubernetes.node.name'], replicaset: ['kubernetes.replicaset.name'], statefulset: ['kubernetes.statefulset.name'], + service: ['kubernetes.service.name'], container: ['kubernetes.container.id'], }, semconv: { diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/service.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/service.ts new file mode 100644 index 0000000000000..4793048aeee02 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/service.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Schema } from '..'; +import { K8sEntity } from '.'; + +export function k8sServiceEntity({ + schema, + name, + uid, + clusterName, + entityId, + ...others +}: { + schema: Schema; + name: string; + uid?: string; + clusterName?: string; + entityId: string; + [key: string]: any; +}) { + if (schema !== 'ecs') { + throw new Error('Schema not supported for service entity: ' + schema); + } + return new K8sEntity(schema, { + 'entity.definition_id': 'service', + 'entity.type': 'service', + 'kubernetes.service.name': name, + 'kubernetes.namespace': clusterName, + 'entity.id': entityId, + ...others, + }); +} diff --git a/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts b/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts index 00d603e17abb8..812c2b91904c4 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts @@ -31,6 +31,7 @@ const CRON_JOB_ENTITY_ID = generateShortId(); const CRON_JOB_UID = generateShortId(); const NODE_ENTITY_ID = generateShortId(); const NODE_UID = generateShortId(); +const SERVICE_UID = generateShortId(); const scenario: Scenario> = async (runOptions) => { const { logger } = runOptions; @@ -45,7 +46,7 @@ const scenario: Scenario> = async (runOptions) => { .interval('1m') .rate(1) .generator((timestamp) => { - return [ + const commonEntities = [ entities.k8s .k8sClusterJobEntity({ schema, @@ -133,6 +134,22 @@ const scenario: Scenario> = async (runOptions) => { }) .timestamp(timestamp), ]; + + if (schema === 'ecs') { + return [ + ...commonEntities, + entities.k8s + .k8sServiceEntity({ + schema, + clusterName: CLUSTER_NAME, + name: 'my_service', + entityId: SERVICE_UID, + }) + .timestamp(timestamp), + ]; + } + + return commonEntities; }); const ecsEntities = getK8sEntitiesEvents('ecs'); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts index 63536fdfcea57..14e2766cac2b2 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts @@ -5,4 +5,4 @@ * 2.0. */ -export const commonEcsIndexPatterns = ['metrics-kubernetes*', 'metrics-apm*', 'logs-*']; +export const commonEcsIndexPatterns = ['metrics-kubernetes*', 'logs-*']; diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts index b7f56c65022f4..e3b830ee19feb 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts @@ -30,6 +30,7 @@ const KUBERNETES_DASHBOARDS_IDS: Record = { [BUILT_IN_ENTITY_TYPES.KUBERNETES.JOB.ecs]: 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013', [BUILT_IN_ENTITY_TYPES.KUBERNETES.NODE.ecs]: 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013', [BUILT_IN_ENTITY_TYPES.KUBERNETES.POD.ecs]: 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013', + [BUILT_IN_ENTITY_TYPES.KUBERNETES.SERVICE.ecs]: 'kubernetes-ff1b3850-bcb1-11ec-b64f-7dd6e8e82013', [BUILT_IN_ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs]: 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013', }; diff --git a/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts b/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts index 167f7b56544ca..db3e91fbf493a 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts @@ -24,6 +24,7 @@ export const BUILT_IN_ENTITY_TYPES = { NAMESPACE: createKubernetesEntity('namespace'), NODE: createKubernetesEntity('node'), POD: createKubernetesEntity('pod'), + SERVICE: createKubernetesEntity('service'), STATEFULSET: createKubernetesEntity('statefulset'), }, } as const; From 2bb2e1e98f1163e429edf6848544e67695483fe1 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Fri, 22 Nov 2024 15:51:58 +0100 Subject: [PATCH 4/4] Revert logic to generate aggregation for displayName; other adjustments --- .../src/lib/entities/index.ts | 2 +- .../src/lib/entities/kubernetes/index.ts | 6 ++++-- .../src/scenarios/k8s_entities.ts | 2 +- .../src/scenarios/many_entities.ts | 2 +- .../entities/built_in/kubernetes/ecs/pod.ts | 9 ++++++++- .../built_in/kubernetes/semconv/cluster.ts | 9 ++++++++- .../built_in/kubernetes/semconv/cron_job.ts | 9 ++++++++- .../built_in/kubernetes/semconv/daemon_set.ts | 9 ++++++++- .../built_in/kubernetes/semconv/deployment.ts | 9 ++++++++- .../built_in/kubernetes/semconv/job.ts | 9 ++++++++- .../built_in/kubernetes/semconv/node.ts | 9 ++++++++- .../built_in/kubernetes/semconv/pod.ts | 9 ++++++++- .../kubernetes/semconv/replica_set.ts | 11 ++++++++-- .../kubernetes/semconv/stateful_set.ts | 9 ++++++++- .../generate_latest_processors.ts | 20 ++++--------------- .../generate_metadata_aggregations.ts | 20 ++++--------------- 16 files changed, 96 insertions(+), 48 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts index 1a2dc7536e366..4b316ebb822e5 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts @@ -24,7 +24,7 @@ import { k8sServiceEntity } from './kubernetes/service'; import { k8sContainerEntity } from './kubernetes/container_entity'; export type EntityDataStreamType = 'metrics' | 'logs' | 'traces'; -export type Schema = 'ecs' | 'semconv'; +export type Schema = 'ecs' | 'otel'; export type EntityFields = Fields & Partial<{ diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts index 19306658341d6..ba6cb1f892f47 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts @@ -24,7 +24,7 @@ const identityFieldsMap: Record> = { service: ['kubernetes.service.name'], container: ['kubernetes.container.id'], }, - semconv: { + otel: { pod: ['k8s.pod.uid'], cluster: ['k8s.cluster.uid'], cronjob: ['k8s.cronjob.uid'], @@ -50,7 +50,9 @@ export class K8sEntity extends Serializable { throw new Error(`Entity definition id not defined`); } - const entityDefinitionWithSchema = `kubernetes_${entityDefinitionId}_${schema}`; + const entityDefinitionWithSchema = `kubernetes_${entityDefinitionId}_${ + schema === 'ecs' ? schema : 'semconv' + }`; const identityFields = identityFieldsMap[schema][entityType]; if (identityFields === undefined || identityFields.length === 0) { throw new Error( diff --git a/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts b/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts index 812c2b91904c4..8300d20589827 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/k8s_entities.ts @@ -153,7 +153,7 @@ const scenario: Scenario> = async (runOptions) => { }); const ecsEntities = getK8sEntitiesEvents('ecs'); - const otelEntities = getK8sEntitiesEvents('semconv'); + const otelEntities = getK8sEntitiesEvents('otel'); return [ withClient( diff --git a/packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts b/packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts index 19817d720a2a5..4aa59d3ee8409 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/many_entities.ts @@ -137,7 +137,7 @@ const scenario: Scenario> = async (runOptions) => { }); const ecsEntities = getK8sEntitiesEvents('ecs'); - const otelEntities = getK8sEntitiesEvents('semconv'); + const otelEntities = getK8sEntitiesEvents('otel'); const synthJavaTraces = entities.serviceEntity({ serviceName: 'synth_java', agentName: ['java'], diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts index 9742f8be00dda..32029617d992c 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts @@ -30,5 +30,12 @@ export const builtInKubernetesPodEcsEntityDefinition: EntityDefinition = frequency: '5m', }, }, - metadata: commonEcsMetadata, + metadata: [ + ...commonEcsMetadata, + { + source: 'kubernetes.pod.name', + destination: 'kubernetes.pod.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/cluster.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/cluster.ts index 0ec244ec617f3..71024cfb166f2 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/cluster.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/cluster.ts @@ -30,5 +30,12 @@ export const builtInKubernetesClusterSemConvEntityDefinition: EntityDefinition = frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.cluster.name', + destination: 'k8s.cluster.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/cron_job.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/cron_job.ts index 6d677943976d1..fff257bcf8e57 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/cron_job.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/cron_job.ts @@ -30,5 +30,12 @@ export const builtInKubernetesCronJobSemConvEntityDefinition: EntityDefinition = frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.cronjob.name', + destination: 'k8s.cronjob.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/daemon_set.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/daemon_set.ts index a4b61933ad316..cf89dcc30e671 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/daemon_set.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/daemon_set.ts @@ -30,5 +30,12 @@ export const builtInKubernetesDaemonSetSemConvEntityDefinition: EntityDefinition frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.daemonset.name', + destination: 'k8s.daemonset.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/deployment.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/deployment.ts index bdb3cb1cef59b..05a89d67ead33 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/deployment.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/deployment.ts @@ -30,5 +30,12 @@ export const builtInKubernetesDeploymentSemConvEntityDefinition: EntityDefinitio frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.deployment.name', + destination: 'k8s.deployment.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/job.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/job.ts index b2e48cf7494fb..557afa54ca55e 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/job.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/job.ts @@ -30,5 +30,12 @@ export const builtInKubernetesJobSemConvEntityDefinition: EntityDefinition = frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.job.name', + destination: 'k8s.job.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/node.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/node.ts index 4b70ea5853fbf..35bbed42e6a4a 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/node.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/node.ts @@ -30,5 +30,12 @@ export const builtInKubernetesNodeSemConvEntityDefinition: EntityDefinition = frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.node.name', + destination: 'k8s.node.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/pod.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/pod.ts index 6dc879d761dd8..05d22163fbacc 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/pod.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/pod.ts @@ -30,5 +30,12 @@ export const builtInKubernetesPodSemConvEntityDefinition: EntityDefinition = frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.pod.name', + destination: 'k8s.pod.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/replica_set.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/replica_set.ts index 7759ce499f920..ff4e33d789da9 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/replica_set.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/replica_set.ts @@ -21,7 +21,7 @@ export const builtInKubernetesReplicaSetSemConvEntityDefinition: EntityDefinitio 'This definition extracts Kubernetes replica set entities using data collected with OpenTelemetry', type: 'k8s.replicaset.otel', indexPatterns: commonOtelIndexPatterns, - identityFields: ['k8s.replicaset.name'], + identityFields: ['k8s.replicaset.uid'], displayNameTemplate: '{{k8s.replicaset.name}}', latest: { timestampField: '@timestamp', @@ -30,5 +30,12 @@ export const builtInKubernetesReplicaSetSemConvEntityDefinition: EntityDefinitio frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.replicaset.name', + destination: 'k8s.replicaset.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/stateful_set.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/stateful_set.ts index c61d7e5d965cd..9c8b385f05c76 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/stateful_set.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/semconv/stateful_set.ts @@ -30,5 +30,12 @@ export const builtInKubernetesStatefulSetSemConvEntityDefinition: EntityDefiniti frequency: '5m', }, }, - metadata: commonOtelMetadata, + metadata: [ + ...commonOtelMetadata, + { + source: 'k8s.statefulset.name', + destination: 'k8s.statefulset.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }, + ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index 633866ddd0008..335c135448b10 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -30,23 +30,11 @@ function mapDestinationToPainless(metadata: MetadataField) { } function createMetadataPainlessScript(definition: EntityDefinition) { - const displayNameField = definition.displayNameTemplate.replace(/[{}]/g, ''); - const shouldAddDisplayNameMetadata = definition.identityFields.some( - ({ field }) => field !== displayNameField - ); - const displayNameMetadata: EntityDefinition['metadata'] = shouldAddDisplayNameMetadata - ? [ - { - source: displayNameField, - destination: displayNameField, - aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, - }, - ] - : undefined; - - const combinedMetadata = [...(definition.metadata ?? []), ...(displayNameMetadata ?? [])]; + if (!definition.metadata) { + return ''; + } - return combinedMetadata.reduce((acc, metadata) => { + return definition.metadata.reduce((acc, metadata) => { const { destination, source } = metadata; const optionalFieldPath = destination.replaceAll('.', '?.'); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 81aa29678e592..796d1e25b55ec 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -8,23 +8,11 @@ import { EntityDefinition } from '@kbn/entities-schema'; export function generateLatestMetadataAggregations(definition: EntityDefinition) { - const displayNameField = definition.displayNameTemplate.replace(/[{}]/g, ''); - const shouldAddDisplayNameMetadata = definition.identityFields.some( - ({ field }) => field !== displayNameField - ); - const displayNameMetadata: EntityDefinition['metadata'] = shouldAddDisplayNameMetadata - ? [ - { - source: displayNameField, - destination: displayNameField, - aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, - }, - ] - : undefined; - - const combinedMetadata = [...(definition.metadata ?? []), ...(displayNameMetadata ?? [])]; + if (!definition.metadata) { + return {}; + } - return combinedMetadata.reduce((aggs, metadata) => { + return definition.metadata.reduce((aggs, metadata) => { const lookbackPeriod = metadata.aggregation.lookbackPeriod || definition.latest.lookbackPeriod; let agg; if (metadata.aggregation.type === 'terms') {