Skip to content

Commit

Permalink
[8.x] [Synthtrace][Inventory] K8s entities support (elastic#197077) (e…
Browse files Browse the repository at this point in the history
…lastic#197293)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Synthtrace][Inventory] K8s entities support
(elastic#197077)](elastic#197077)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Cauê
Marcondes","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-22T15:43:29Z","message":"[Synthtrace][Inventory]
K8s entities support (elastic#197077)\n\nPart of
https://github.com/elastic/kibana/issues/196155\r\n\r\nNew Synthtrace
scenario
created:\r\n[k8s_entities.ts](https://github.com/elastic/kibana/pull/197077/files#diff-33a935a5fb8848d743e40d89b018ed8ac82ae992c0778cd6cea072d091aa0647)\r\n\r\n```\r\nnode
scripts/synthtrace k8s_entities.ts --clean
--live\r\n```\r\n\r\n\r\nCluster sample data:\r\n```\r\n {\r\n
\"_index\":
\".entities.v1.latest.builtin_kubernetes_cluster_ecss_from_ecs_data\",\r\n
\"_id\": \"2060900000000000\",\r\n \"_score\": 1,\r\n \"_source\": {\r\n
\"entity\": {\r\n \"type\": \"kubernetes_cluster_ecs\",\r\n \"id\":
\"2060900000000000\",\r\n \"definitionId\":
\"builtin_kubernetes_cluster_ecs\",\r\n \"displayName\":
\"cluster_foo\",\r\n \"lastSeenTimestamp\":
\"2024-10-21T16:15:17.570Z\"\r\n },\r\n \"orchestrator\": {\r\n
\"cluster\": {\r\n \"name\": \"cluster_foo\"\r\n }\r\n },\r\n \"event\":
{\r\n \"ingested\": \"2024-10-21T16:15:17.570Z\"\r\n }\r\n }\r\n },\r\n
{\r\n \"_index\":
\".entities.v1.latest.builtin_kubernetes_cluster_semconvs_from_ecs_data\",\r\n
\"_id\": \"2060900000000000\",\r\n \"_score\": 1,\r\n \"_source\": {\r\n
\"entity\": {\r\n \"type\": \"kubernetes_cluster_semconv\",\r\n \"id\":
\"2060900000000000\",\r\n \"definitionId\":
\"builtin_kubernetes_cluster_semconv\",\r\n \"displayName\":
\"cluster_foo\",\r\n \"lastSeenTimestamp\":
\"2024-10-21T16:15:17.570Z\"\r\n },\r\n \"k8s\": {\r\n \"cluster\":
{\r\n \"uid\": \"cluster_foo\"\r\n }\r\n },\r\n \"event\": {\r\n
\"ingested\": \"2024-10-21T16:15:17.570Z\"\r\n }\r\n }\r\n
},\r\n```","sha":"5e40320d9fe2d2c0b676071edf011c5831d43662","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","ci:project-deploy-observability","Team:obs-ux-infra_services","v8.17.0"],"title":"[Synthtrace][Inventory]
K8s entities
support","number":197077,"url":"https://github.com/elastic/kibana/pull/197077","mergeCommit":{"message":"[Synthtrace][Inventory]
K8s entities support (elastic#197077)\n\nPart of
https://github.com/elastic/kibana/issues/196155\r\n\r\nNew Synthtrace
scenario
created:\r\n[k8s_entities.ts](https://github.com/elastic/kibana/pull/197077/files#diff-33a935a5fb8848d743e40d89b018ed8ac82ae992c0778cd6cea072d091aa0647)\r\n\r\n```\r\nnode
scripts/synthtrace k8s_entities.ts --clean
--live\r\n```\r\n\r\n\r\nCluster sample data:\r\n```\r\n {\r\n
\"_index\":
\".entities.v1.latest.builtin_kubernetes_cluster_ecss_from_ecs_data\",\r\n
\"_id\": \"2060900000000000\",\r\n \"_score\": 1,\r\n \"_source\": {\r\n
\"entity\": {\r\n \"type\": \"kubernetes_cluster_ecs\",\r\n \"id\":
\"2060900000000000\",\r\n \"definitionId\":
\"builtin_kubernetes_cluster_ecs\",\r\n \"displayName\":
\"cluster_foo\",\r\n \"lastSeenTimestamp\":
\"2024-10-21T16:15:17.570Z\"\r\n },\r\n \"orchestrator\": {\r\n
\"cluster\": {\r\n \"name\": \"cluster_foo\"\r\n }\r\n },\r\n \"event\":
{\r\n \"ingested\": \"2024-10-21T16:15:17.570Z\"\r\n }\r\n }\r\n },\r\n
{\r\n \"_index\":
\".entities.v1.latest.builtin_kubernetes_cluster_semconvs_from_ecs_data\",\r\n
\"_id\": \"2060900000000000\",\r\n \"_score\": 1,\r\n \"_source\": {\r\n
\"entity\": {\r\n \"type\": \"kubernetes_cluster_semconv\",\r\n \"id\":
\"2060900000000000\",\r\n \"definitionId\":
\"builtin_kubernetes_cluster_semconv\",\r\n \"displayName\":
\"cluster_foo\",\r\n \"lastSeenTimestamp\":
\"2024-10-21T16:15:17.570Z\"\r\n },\r\n \"k8s\": {\r\n \"cluster\":
{\r\n \"uid\": \"cluster_foo\"\r\n }\r\n },\r\n \"event\": {\r\n
\"ingested\": \"2024-10-21T16:15:17.570Z\"\r\n }\r\n }\r\n
},\r\n```","sha":"5e40320d9fe2d2c0b676071edf011c5831d43662"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/197077","number":197077,"mergeCommit":{"message":"[Synthtrace][Inventory]
K8s entities support (elastic#197077)\n\nPart of
https://github.com/elastic/kibana/issues/196155\r\n\r\nNew Synthtrace
scenario
created:\r\n[k8s_entities.ts](https://github.com/elastic/kibana/pull/197077/files#diff-33a935a5fb8848d743e40d89b018ed8ac82ae992c0778cd6cea072d091aa0647)\r\n\r\n```\r\nnode
scripts/synthtrace k8s_entities.ts --clean
--live\r\n```\r\n\r\n\r\nCluster sample data:\r\n```\r\n {\r\n
\"_index\":
\".entities.v1.latest.builtin_kubernetes_cluster_ecss_from_ecs_data\",\r\n
\"_id\": \"2060900000000000\",\r\n \"_score\": 1,\r\n \"_source\": {\r\n
\"entity\": {\r\n \"type\": \"kubernetes_cluster_ecs\",\r\n \"id\":
\"2060900000000000\",\r\n \"definitionId\":
\"builtin_kubernetes_cluster_ecs\",\r\n \"displayName\":
\"cluster_foo\",\r\n \"lastSeenTimestamp\":
\"2024-10-21T16:15:17.570Z\"\r\n },\r\n \"orchestrator\": {\r\n
\"cluster\": {\r\n \"name\": \"cluster_foo\"\r\n }\r\n },\r\n \"event\":
{\r\n \"ingested\": \"2024-10-21T16:15:17.570Z\"\r\n }\r\n }\r\n },\r\n
{\r\n \"_index\":
\".entities.v1.latest.builtin_kubernetes_cluster_semconvs_from_ecs_data\",\r\n
\"_id\": \"2060900000000000\",\r\n \"_score\": 1,\r\n \"_source\": {\r\n
\"entity\": {\r\n \"type\": \"kubernetes_cluster_semconv\",\r\n \"id\":
\"2060900000000000\",\r\n \"definitionId\":
\"builtin_kubernetes_cluster_semconv\",\r\n \"displayName\":
\"cluster_foo\",\r\n \"lastSeenTimestamp\":
\"2024-10-21T16:15:17.570Z\"\r\n },\r\n \"k8s\": {\r\n \"cluster\":
{\r\n \"uid\": \"cluster_foo\"\r\n }\r\n },\r\n \"event\": {\r\n
\"ingested\": \"2024-10-21T16:15:17.570Z\"\r\n }\r\n }\r\n
},\r\n```","sha":"5e40320d9fe2d2c0b676071edf011c5831d43662"}},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Cauê Marcondes <[email protected]>
  • Loading branch information
kibanamachine and cauemarcondes authored Nov 1, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 336a924 commit d74f8a7
Showing 14 changed files with 714 additions and 2 deletions.
29 changes: 28 additions & 1 deletion packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts
Original file line number Diff line number Diff line change
@@ -11,8 +11,19 @@ import { Fields } from '../entity';
import { serviceEntity } from './service_entity';
import { hostEntity } from './host_entity';
import { containerEntity } from './container_entity';
import { k8sClusterJobEntity } from './kubernetes/cluster_entity';
import { k8sCronJobEntity } from './kubernetes/cron_job_entity';
import { k8sDaemonSetEntity } from './kubernetes/daemon_set_entity';
import { k8sDeploymentEntity } from './kubernetes/deployment_entity';
import { k8sJobSetEntity } from './kubernetes/job_set_entity';
import { k8sNodeEntity } from './kubernetes/node_entity';
import { k8sPodEntity } from './kubernetes/pod_entity';
import { k8sReplicaSetEntity } from './kubernetes/replica_set';
import { k8sStatefulSetEntity } from './kubernetes/stateful_set';
import { k8sContainerEntity } from './kubernetes/container_entity';

export type EntityDataStreamType = 'metrics' | 'logs' | 'traces';
export type Schema = 'ecs' | 'semconv';

export type EntityFields = Fields &
Partial<{
@@ -32,4 +43,20 @@ export type EntityFields = Fields &
[key: string]: any;
}>;

export const entities = { serviceEntity, hostEntity, containerEntity };
export const entities = {
serviceEntity,
hostEntity,
containerEntity,
k8s: {
k8sClusterJobEntity,
k8sCronJobEntity,
k8sDaemonSetEntity,
k8sDeploymentEntity,
k8sJobSetEntity,
k8sNodeEntity,
k8sPodEntity,
k8sReplicaSetEntity,
k8sStatefulSetEntity,
k8sContainerEntity,
},
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sClusterJobEntity({
schema,
name,
entityId,
...others
}: {
schema: Schema;
name: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'cluster',
'orchestrator.cluster.name': name,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'cluster',
'k8s.cluster.uid': name,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sContainerEntity({
schema,
id,
entityId,
...others
}: {
schema: Schema;
id: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'container',
'kubernetes.container.id': id,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'container',
'container.id': id,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sCronJobEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'cron_job',
'kubernetes.cronjob.name': name,
'kubernetes.cronjob.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'cron_job',
'k8s.cronjob.name': name,
'k8s.cronjob.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sDaemonSetEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'daemon_set',
'kubernetes.daemonset.name': name,
'kubernetes.daemonset.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'daemon_set',
'k8s.daemonset.name': name,
'k8s.daemonset.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sDeploymentEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'deployment',
'kubernetes.deployment.name': name,
'kubernetes.deployment.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'deployment',
'k8s.deployment.name': name,
'k8s.deployment.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import type { EntityFields, Schema } from '..';
import { Serializable } from '../../serializable';

const identityFieldsMap: Record<Schema, Record<string, string[]>> = {
ecs: {
pod: ['kubernetes.pod.name'],
cluster: ['orchestrator.cluster.name'],
cron_job: ['kubernetes.cronjob.name'],
daemon_set: ['kubernetes.daemonset.name'],
deployment: ['kubernetes.deployment.name'],
job: ['kubernetes.job.name'],
node: ['kubernetes.node.name'],
replica_set: ['kubernetes.replicaset.name'],
stateful_set: ['kubernetes.statefulset.name'],
container: ['kubernetes.container.id'],
},
semconv: {
pod: ['k8s.pod.name'],
cluster: ['k8s.cluster.uid'],
cron_job: ['k8s.cronjob.name'],
daemon_set: ['k8s.daemonset.name'],
deployment: ['k8s.deployment.name'],
job: ['k8s.job.name'],
node: ['k8s.node.uid'],
replica_set: ['k8s.replicaset.name'],
stateful_set: ['k8s.statefulset.name'],
container: ['container.id'],
},
};

export class K8sEntity extends Serializable<EntityFields> {
constructor(schema: Schema, fields: EntityFields) {
const entityType = fields['entity.type'];
if (entityType === undefined) {
throw new Error(`Entity type not defined: ${entityType}`);
}

const entityTypeWithSchema = `kubernetes_${entityType}_${schema}`;
const identityFields = identityFieldsMap[schema][entityType];
if (identityFields === undefined || identityFields.length === 0) {
throw new Error(
`Identity fields not defined for schema: ${schema} and entity type: ${entityType}`
);
}

super({
...fields,
'entity.type': entityTypeWithSchema,
'entity.definitionId': `builtin_${entityTypeWithSchema}`,
'entity.identityFields': identityFields,
'entity.displayName': getDisplayName({ identityFields, fields }),
});
}
}

function getDisplayName({
identityFields,
fields,
}: {
identityFields: string[];
fields: EntityFields;
}) {
return identityFields
.map((field) => fields[field])
.filter((_) => _)
.join(':');
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sJobSetEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'job',
'kubernetes.job.name': name,
'kubernetes.job.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'job',
'k8s.job.name': name,
'k8s.job.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}
Loading

0 comments on commit d74f8a7

Please sign in to comment.