Skip to content

Commit

Permalink
Inventory k8s entities fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
crespocarlos committed Nov 21, 2024
1 parent d88ce9e commit ac4eebc
Show file tree
Hide file tree
Showing 22 changed files with 108 additions and 59 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
* 2.0.
*/

export const commonEcsIndexPatterns = ['metrics-kubernetes*', 'logs-*'];
export const commonEcsIndexPatterns = ['metrics-kubernetes*', 'metrics-apm*', 'logs-*'];
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import { commonEcsMetadata } from '../common/ecs_metadata';
export const builtInKubernetesCronJobEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_cron_job_ecs`,
filter: 'kubernetes.cronjob.uid : *',
filter: 'kubernetes.cronjob.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes CronJob from ECS data',
description:
'This definition extracts Kubernetes cron job entities from the Kubernetes integration data streams',
type: 'k8s.cronjob.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.cronjob.uid'],
identityFields: ['kubernetes.cronjob.name'],
displayNameTemplate: '{{kubernetes.cronjob.name}}',
latest: {
timestampField: '@timestamp',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata';
export const builtInKubernetesDaemonSetEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_daemon_set_ecs`,
filter: 'kubernetes.daemonset.uid : *',
filter: 'kubernetes.daemonset.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes DaemonSet from ECS data',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
export const builtInKubernetesDeploymentEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_deployment_ecs`,
filter: 'kubernetes.deployment.uid : *',
filter: 'kubernetes.deployment.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes Deployment from ECS data',
description:
'This definition extracts Kubernetes deployment entities from the Kubernetes integration data streams',
type: 'k8s.deployment.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.deployment.uid'],
identityFields: ['kubernetes.deployment.name'],
displayNameTemplate: '{{kubernetes.deployment.name}}',
latest: {
timestampField: '@timestamp',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import { commonEcsMetadata } from '../common/ecs_metadata';
export const builtInKubernetesJobEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_job_ecs`,
filter: 'kubernetes.job.uid : *',
filter: 'kubernetes.job.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes Job from ECS data',
description:
'This definition extracts Kubernetes job entities from the Kubernetes integration data streams',
type: 'k8s.job.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.job.uid'],
identityFields: ['kubernetes.job.name'],
displayNameTemplate: '{{kubernetes.job.name}}',
latest: {
timestampField: '@timestamp',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import { commonEcsMetadata } from '../common/ecs_metadata';
export const builtInKubernetesNodeEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_node_ecs`,
filer: 'kubernetes.node.uid : *',
filer: 'kubernetes.node.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes Node from ECS data',
description:
'This definition extracts Kubernetes node entities from the Kubernetes integration data streams',
type: 'k8s.node.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.node.uid'],
identityFields: ['kubernetes.node.name'],
displayNameTemplate: '{{kubernetes.node.name}}',
latest: {
timestampField: '@timestamp',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export const builtInKubernetesPodEcsEntityDefinition: EntityDefinition =
'This definition extracts Kubernetes pod entities from the Kubernetes integration data streams',
type: 'k8s.pod.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.pod.name'],
identityFields: ['kubernetes.pod.uid'],
displayNameTemplate: '{{kubernetes.pod.name}}',
latest: {
timestampField: '@timestamp',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,15 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
export const builtInKubernetesReplicaSetEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_replica_set_ecs`,
filer: 'kubernetes.replicaset.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes ReplicaSet from ECS data',
description:
'This definition extracts Kubernetes replica set entities from the Kubernetes integration data streams',
type: 'k8s.replicaset.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.replicaset.uid'],
identityFields: ['kubernetes.replicaset.name'],
displayNameTemplate: '{{kubernetes.replicaset.name}}',
latest: {
timestampField: '@timestamp',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
export const builtInKubernetesStatefulSetEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_stateful_set_ecs`,
filter: 'kubernetes.statefulset.uid : *',
filter: 'kubernetes.statefulset.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes StatefulSet from ECS data',
description:
'This definition extracts Kubernetes stateful set entities from the Kubernetes integration data streams',
type: 'k8s.statefulset.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.statefulset.uid'],
identityFields: ['kubernetes.statefulset.name'],
displayNameTemplate: '{{kubernetes.statefulset.name}}',
latest: {
timestampField: '@timestamp',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export const builtInKubernetesNodeSemConvEntityDefinition: EntityDefinition =
type: 'k8s.node.otel',
indexPatterns: commonOtelIndexPatterns,
identityFields: ['k8s.node.uid'],
displayNameTemplate: '{{k8s.node.uid}}',
displayNameTemplate: '{{k8s.node.name}}',
latest: {
timestampField: '@timestamp',
lookbackPeriod: '10m',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export const builtInKubernetesReplicaSetSemConvEntityDefinition: EntityDefinitio
name: 'Kubernetes ReplicaSet from SemConv data',
description:
'This definition extracts Kubernetes replica set entities using data collected with OpenTelemetry',
type: 'kubernetes_replica_set_semconv',
type: 'k8s.replicaset.otel',
indexPatterns: commonOtelIndexPatterns,
identityFields: ['k8s.replicaset.name'],
displayNameTemplate: '{{k8s.replicaset.name}}',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,23 @@ function mapDestinationToPainless(metadata: MetadataField) {
}

function createMetadataPainlessScript(definition: EntityDefinition) {
if (!definition.metadata) {
return '';
}
const displayNameField = definition.displayNameTemplate.replace(/[{}]/g, '');
const shouldAddDisplayNameMetadata = definition.identityFields.some(
({ field }) => field !== displayNameField
);
const displayNameMetadata: EntityDefinition['metadata'] = shouldAddDisplayNameMetadata
? [
{
source: displayNameField,
destination: displayNameField,
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
]
: undefined;

const combinedMetadata = [...(definition.metadata ?? []), ...(displayNameMetadata ?? [])];

return definition.metadata.reduce((acc, metadata) => {
return combinedMetadata.reduce((acc, metadata) => {
const { destination, source } = metadata;
const optionalFieldPath = destination.replaceAll('.', '?.');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,23 @@
import { EntityDefinition } from '@kbn/entities-schema';

export function generateLatestMetadataAggregations(definition: EntityDefinition) {
if (!definition.metadata) {
return {};
}
const displayNameField = definition.displayNameTemplate.replace(/[{}]/g, '');
const shouldAddDisplayNameMetadata = definition.identityFields.some(
({ field }) => field !== displayNameField
);
const displayNameMetadata: EntityDefinition['metadata'] = shouldAddDisplayNameMetadata
? [
{
source: displayNameField,
destination: displayNameField,
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
]
: undefined;

const combinedMetadata = [...(definition.metadata ?? []), ...(displayNameMetadata ?? [])];

return definition.metadata.reduce((aggs, metadata) => {
return combinedMetadata.reduce((aggs, metadata) => {
const lookbackPeriod = metadata.aggregation.lookbackPeriod || definition.latest.lookbackPeriod;
let agg;
if (metadata.aggregation.type === 'terms') {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,16 @@
*/

import * as z from '@kbn/zod';
import { EntityDataStreamType, ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import {
EntityDataStreamType,
BUILT_IN_ENTITY_TYPES,
} from '@kbn/observability-shared-plugin/common';
import { useFetcher } from '../../../hooks/use_fetcher';

const EntityTypeSchema = z.union([z.literal(ENTITY_TYPES.HOST), z.literal(ENTITY_TYPES.CONTAINER)]);
const EntityTypeSchema = z.union([
z.literal(BUILT_IN_ENTITY_TYPES.HOST),
z.literal(BUILT_IN_ENTITY_TYPES.CONTAINER),
]);
const EntityDataStreamSchema = z.union([
z.literal(EntityDataStreamType.METRICS),
z.literal(EntityDataStreamType.LOGS),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
import { getFieldByType } from '@kbn/metrics-data-access-plugin/common';
import { decodeOrThrow } from '@kbn/io-ts-utils';
import useLocalStorage from 'react-use/lib/useLocalStorage';
import { ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import { BUILT_IN_ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import { useSourceContext } from '../../../../containers/metrics_source';
import { isPending, useFetcher } from '../../../../hooks/use_fetcher';
import { parseSearchString } from './parse_search_string';
Expand Down Expand Up @@ -58,7 +58,7 @@ export const Processes = () => {
const { request$ } = useRequestObservable();
const { isActiveTab } = useTabSwitcherContext();
const { dataStreams, status: dataStreamsStatus } = useEntitySummary({
entityType: ENTITY_TYPES.HOST,
entityType: BUILT_IN_ENTITY_TYPES.HOST,
entityId: asset.name,
});
const addMetricsCalloutId: AddMetricsCalloutKey = 'hostProcesses';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { schema } from '@kbn/config-schema';
import { METRICS_APP_ID } from '@kbn/deeplinks-observability/constants';
import { entityCentricExperience } from '@kbn/observability-plugin/common';
import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client';
import { ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import { BUILT_IN_ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import { getInfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client';
import { InfraBackendLibs } from '../../lib/infra_types';
import { getDataStreamTypes } from './get_data_stream_types';
Expand All @@ -24,8 +24,8 @@ export const initEntitiesConfigurationRoutes = (libs: InfraBackendLibs) => {
validate: {
params: schema.object({
entityType: schema.oneOf([
schema.literal(ENTITY_TYPES.HOST),
schema.literal(ENTITY_TYPES.CONTAINER),
schema.literal(BUILT_IN_ENTITY_TYPES.HOST),
schema.literal(BUILT_IN_ENTITY_TYPES.CONTAINER),
]),
entityId: schema.string(),
}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export function EntityIcon({ entity }: EntityIconProps) {
return <AgentIcon agentName={castArray(entity.agent?.name)[0]} role="presentation" />;
}

if (entity.entityType.startsWith('kubernetes')) {
if (entity.entityType.startsWith('k8s')) {
return <EuiIcon type="logoKubernetes" size="l" />;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { useDetailViewRedirect } from './use_detail_view_redirect';
import { useKibana } from './use_kibana';
import {
CONTAINER_ID,
ENTITY_TYPES,
BUILT_IN_ENTITY_TYPES,
HOST_NAME,
SERVICE_NAME,
} from '@kbn/observability-shared-plugin/common';
Expand Down Expand Up @@ -134,15 +134,30 @@ describe('useDetailViewRedirect', () => {
});

[
[ENTITY_TYPES.KUBERNETES.CLUSTER.ecs, 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c'],
[ENTITY_TYPES.KUBERNETES.CLUSTER.semconv, 'kubernetes_otel-cluster-overview'],
[ENTITY_TYPES.KUBERNETES.CRONJOB.ecs, 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs, 'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs, 'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.JOB.ecs, 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.NODE.ecs, 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.POD.ecs, 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs, 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013'],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.ecs,
'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c',
],
[BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.semconv, 'kubernetes_otel-cluster-overview'],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.CRONJOB.ecs,
'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013',
],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs,
'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013',
],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs,
'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013',
],
[BUILT_IN_ENTITY_TYPES.KUBERNETES.JOB.ecs, 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013'],
[BUILT_IN_ENTITY_TYPES.KUBERNETES.NODE.ecs, 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013'],
[BUILT_IN_ENTITY_TYPES.KUBERNETES.POD.ecs, 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013'],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs,
'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013',
],
].forEach(([entityType, dashboardId]) => {
it(`getEntityRedirectUrl should return the correct URL for ${entityType} entity`, () => {
const entity: InventoryEntity = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*/
import {
ASSET_DETAILS_LOCATOR_ID,
ENTITY_TYPES,
BUILT_IN_ENTITY_TYPES,
SERVICE_OVERVIEW_LOCATOR_ID,
type AssetDetailsLocatorParams,
type ServiceOverviewParams,
Expand All @@ -20,15 +20,18 @@ import type { InventoryEntity } from '../../common/entities';
import { useKibana } from './use_kibana';

const KUBERNETES_DASHBOARDS_IDS: Record<string, string> = {
[ENTITY_TYPES.KUBERNETES.CLUSTER.ecs]: 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c',
[ENTITY_TYPES.KUBERNETES.CLUSTER.semconv]: 'kubernetes_otel-cluster-overview',
[ENTITY_TYPES.KUBERNETES.CRONJOB.ecs]: 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs]: 'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs]: 'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.JOB.ecs]: 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.NODE.ecs]: 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.POD.ecs]: 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs]: 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.ecs]: 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.semconv]: 'kubernetes_otel-cluster-overview',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.CRONJOB.ecs]: 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs]:
'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs]:
'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.JOB.ecs]: 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.NODE.ecs]: 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.POD.ecs]: 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs]:
'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013',
};

export const useDetailViewRedirect = () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,24 @@
*/

const createKubernetesEntity = <T extends string>(base: T) => ({
ecs: `kubernetes_${base}_ecs` as const,
semconv: `kubernetes_${base}_semconv` as const,
ecs: `k8s.${base}.ecs` as const,
semconv: `k8s.${base}.semconv` as const,
});

export const ENTITY_TYPES = {
export const BUILT_IN_ENTITY_TYPES = {
HOST: 'host',
CONTAINER: 'container',
SERVICE: 'service',
KUBERNETES: {
CLUSTER: createKubernetesEntity('cluster'),
CONTAINER: createKubernetesEntity('container'),
CRONJOB: createKubernetesEntity('cron_job'),
DAEMONSET: createKubernetesEntity('daemon_set'),
DAEMONSET: createKubernetesEntity('daemonset'),
DEPLOYMENT: createKubernetesEntity('deployment'),
JOB: createKubernetesEntity('job'),
NAMESPACE: createKubernetesEntity('namespace'),
NODE: createKubernetesEntity('node'),
POD: createKubernetesEntity('pod'),
STATEFULSET: createKubernetesEntity('stateful_set'),
STATEFULSET: createKubernetesEntity('statefulset'),
},
} as const;
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,5 @@
* 2.0.
*/

export { ENTITY_TYPES } from './entity_types';
export { BUILT_IN_ENTITY_TYPES } from './entity_types';
export { EntityDataStreamType } from './entity_data_stream_types';
Original file line number Diff line number Diff line change
Expand Up @@ -219,4 +219,4 @@ export {

export { COMMON_OBSERVABILITY_GROUPING } from './embeddable_grouping';

export { ENTITY_TYPES, EntityDataStreamType } from './entity';
export { BUILT_IN_ENTITY_TYPES, EntityDataStreamType } from './entity';

0 comments on commit ac4eebc

Please sign in to comment.