Skip to content

Commit

Permalink
[8.x] [Inventory] Add k8s fields to Service entity type (#195407) (#1…
Browse files Browse the repository at this point in the history 
…95601)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Inventory] Add k8s fields to Service entity type
(#195407)](#195407)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Cauê
Marcondes","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-09T13:44:12Z","message":"[Inventory]
Add k8s fields to Service entity type (#195407)\n\ncloses
https://github.com/elastic/kibana/issues/195244\r\n\r\n- Removed metrics
definition from service, host and container\r\n- Removed `metrics-apm`
index patterns from the service definition\r\nbecause k8s fields are not
available on that scope.\r\n- Added `traces-apm*` index pattern on the
service
definition\r\n\r\n\r\nhttps://github.com/user-attachments/assets/6c6b4fd6-817a-494e-8649-e2d76a8e98e3","sha":"cc7fdba1422f2717984b958509be13abc820b15b","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","ci:cloud-deploy","ci:project-deploy-observability","Team:obs-ux-infra_services","v8.16.0","Feature:EEM"],"title":"[Inventory]
Add k8s fields to Service entity
type","number":195407,"url":"https://github.com/elastic/kibana/pull/195407","mergeCommit":{"message":"[Inventory]
Add k8s fields to Service entity type (#195407)\n\ncloses
https://github.com/elastic/kibana/issues/195244\r\n\r\n- Removed metrics
definition from service, host and container\r\n- Removed `metrics-apm`
index patterns from the service definition\r\nbecause k8s fields are not
available on that scope.\r\n- Added `traces-apm*` index pattern on the
service
definition\r\n\r\n\r\nhttps://github.com/user-attachments/assets/6c6b4fd6-817a-494e-8649-e2d76a8e98e3","sha":"cc7fdba1422f2717984b958509be13abc820b15b"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195407","number":195407,"mergeCommit":{"message":"[Inventory]
Add k8s fields to Service entity type (#195407)\n\ncloses
https://github.com/elastic/kibana/issues/195244\r\n\r\n- Removed metrics
definition from service, host and container\r\n- Removed `metrics-apm`
index patterns from the service definition\r\nbecause k8s fields are not
available on that scope.\r\n- Added `traces-apm*` index pattern on the
service
definition\r\n\r\n\r\nhttps://github.com/user-attachments/assets/6c6b4fd6-817a-494e-8649-e2d76a8e98e3","sha":"cc7fdba1422f2717984b958509be13abc820b15b"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Cauê Marcondes <[email protected]>
  • Loading branch information
@kibanamachine @cauemarcondes
kibanamachine and cauemarcondes authored Oct 9, 2024
1 parent 062ea57 commit ef6f776
Show file tree
Hide file tree
Showing 16 changed files with 12 additions and 1,161 deletions.
92 changes: 1 addition & 91 deletions x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}containers_from_ecs_data`,
managed: true,
version: '1.0.0',
version: '0.1.0',
name: 'Containers from ECS data',
description:
'This definition extracts container entities from common data streams by looking for the ECS field container.id',
Expand Down Expand Up @@ -65,94 +65,4 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition =
'agent.type',
'agent.ephemeral_id',
],
metrics: [
{
name: 'log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'log.level: * OR error.log.level: *',
},
],
},
{
name: 'error_log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")',
},
],
},
{
name: 'cpu_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.cpu.total.pct',
},
],
},
{
name: 'memory_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.memory.usage.pct',
},
],
},
{
name: 'network_in_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.network.in.bytes',
},
],
},
{
name: 'network_out_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.network.out.bytes',
},
],
},
{
name: 'disk_read_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.diskio.read.ops',
},
],
},
{
name: 'disk_write_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.diskio.write.ops',
},
],
},
],
});
113 changes: 1 addition & 112 deletions x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { BUILT_IN_ID_PREFIX } from './constants';
export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}hosts_from_ecs_data`,
managed: true,
version: '1.0.0',
version: '0.1.0',
name: 'Hosts from ECS data',
description:
'This definition extracts host entities from common data streams by looking for the ECS field host.name',
Expand Down Expand Up @@ -65,115 +65,4 @@ export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefin
'agent.type',
'agent.version',
],
metrics: [
{
name: 'log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'log.level: * OR error.log.level: *',
},
],
},
{
name: 'error_log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")',
},
],
},
{
name: 'cpu_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.cpu.total.norm.pct',
},
],
},
{
name: 'normalized_load_avg',
equation: 'A / B',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.load.1',
},
{
name: 'B',
aggregation: 'max',
field: 'system.load.cores',
},
],
},
{
name: 'memory_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.memory.actual.used.pct',
},
],
},
{
name: 'memory_free_avg',
equation: 'A - B',
metrics: [
{
name: 'A',
aggregation: 'max',
field: 'system.memory.total',
},
{
name: 'B',
aggregation: 'avg',
field: 'system.memory.actual.used.bytes',
},
],
},
{
name: 'disk_usage_max',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'max',
field: 'system.filesystem.used.pct',
},
],
},
{
name: 'rx_avg',
equation: 'A * 8',
metrics: [
{
name: 'A',
aggregation: 'sum',
field: 'host.network.ingress.bytes',
},
],
},
{
name: 'tx_avg',
equation: 'A * 8',
metrics: [
{
name: 'A',
aggregation: 'sum',
field: 'host.network.egress.bytes',
},
],
},
],
});
90 changes: 6 additions & 84 deletions x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,16 @@
import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
import { BUILT_IN_ID_PREFIX } from './constants';

const serviceTransactionFilter = (additionalFilters: string[] = []) => {
const baseFilters = [
'processor.event: "metric"',
'metricset.name: "service_transaction"',
'metricset.interval: "1m"',
];

return [...baseFilters, ...additionalFilters].join(' AND ');
};

export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
version: '0.3.0',
version: '0.4.0',
id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`,
name: 'Services from ECS data',
description:
'This definition extracts service entities from common data streams by looking for the ECS field service.name',
type: 'service',
managed: true,
indexPatterns: [
'logs-*',
'filebeat*',
'metrics-apm.service_transaction.1m*',
'metrics-apm.service_summary.1m*',
],
indexPatterns: ['logs-*', 'filebeat*', 'traces-apm*'],
history: {
timestampField: '@timestamp',
interval: '1m',
Expand Down Expand Up @@ -65,72 +50,9 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
'cloud.provider',
'cloud.availability_zone',
'cloud.machine.type',
],
metrics: [
{
name: 'latency',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
filter: serviceTransactionFilter(),
field: 'transaction.duration.histogram',
},
],
},
{
name: 'throughput',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'value_count',
filter: serviceTransactionFilter(),
field: 'transaction.duration.summary',
},
],
},
{
name: 'failedTransactionRate',
equation: '1 - (A / B)',
metrics: [
{
name: 'A',
aggregation: 'sum',
filter: serviceTransactionFilter(),
field: 'event.success_count',
},
{
name: 'B',
aggregation: 'value_count',
filter: serviceTransactionFilter(),
field: 'event.success_count',
},
],
},
{
name: 'logErrorRate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter:
'log.level: "error" OR log.level: "ERROR" OR error.log.level: "error" OR error.log.level: "ERROR"',
},
],
},
{
name: 'logRate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'data_stream.type: logs',
},
],
},
'kubernetes.namespace',
'orchestrator.cluster.name',
'k8s.namespace.name',
'k8s.cluster.name',
],
});
13 changes: 0 additions & 13 deletions x-pack/plugins/observability_solution/apm/common/entities/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,3 @@ export enum EntityDataStreamType {
TRACES = 'traces',
LOGS = 'logs',
}

interface TraceMetrics {
latency?: number | null;
throughput?: number | null;
failedTransactionRate?: number | null;
}

interface LogsMetrics {
logRate?: number | null;
logErrorRate?: number | null;
}

export type EntityMetrics = TraceMetrics & LogsMetrics;
28 changes: 1 addition & 27 deletions x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import {
import type { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
import { getEntityLatestServices } from './get_entity_latest_services';
import type { EntityLatestServiceRaw } from './types';
import { getEntityHistoryServicesMetrics } from './get_entity_history_services_metrics';

export function entitiesRangeQuery(start?: number, end?: number): QueryDslQueryContainer[] {
if (!start || !end) {
Expand Down Expand Up @@ -64,30 +63,5 @@ export async function getEntities({
serviceName,
});

const serviceEntitiesHistoryMetricsMap = entityLatestServices.length
? await getEntityHistoryServicesMetrics({
start,
end,
entitiesESClient,
entityIds: entityLatestServices.map((latestEntity) => latestEntity.entity.id),
size,
})
: undefined;

return entityLatestServices.map((latestEntity) => {
const historyEntityMetrics = serviceEntitiesHistoryMetricsMap?.[latestEntity.entity.id];
return {
...latestEntity,
entity: {
...latestEntity.entity,
metrics: historyEntityMetrics || {
latency: undefined,
logErrorRate: undefined,
failedTransactionRate: undefined,
logRate: undefined,
throughput: undefined,
},
},
};
});
return entityLatestServices;
}
Loading

0 comments on commit ef6f776

Please sign in to comment.