diff --git a/x-pack/plugins/fleet/server/collectors/agents_per_output.test.ts b/x-pack/plugins/fleet/server/collectors/agents_per_output.test.ts index 1202876c754a8..f68a700865708 100644 --- a/x-pack/plugins/fleet/server/collectors/agents_per_output.test.ts +++ b/x-pack/plugins/fleet/server/collectors/agents_per_output.test.ts @@ -19,11 +19,13 @@ jest.mock('../services', () => { { agents: 1, data_output_id: 'logstash1' }, { agents: 1, monitoring_output_id: 'kafka1' }, { agents: 1, data_output_id: 'elasticsearch2', monitoring_output_id: 'elasticsearch2' }, + { agents: 1, data_output_id: 'elasticsearch3', monitoring_output_id: 'elasticsearch3' }, { agents: 1, data_output_id: 'es-containerhost', monitoring_output_id: 'es-containerhost', }, + { agents: 1, data_output_id: 'remote-es', monitoring_output_id: 'remote-es' }, ], }), }, @@ -35,11 +37,14 @@ jest.mock('../services', () => { is_default: true, is_default_monitoring: true, type: 'elasticsearch', + preset: 'balanced', }, { id: 'logstash1', type: 'logstash' }, { id: 'kafka1', type: 'kafka' }, - { id: 'elasticsearch2', type: 'elasticsearch' }, - { id: 'es-containerhost', type: 'elasticsearch' }, + { id: 'elasticsearch2', type: 'elasticsearch', preset: 'custom' }, + { id: 'elasticsearch3', type: 'elasticsearch', preset: 'balanced' }, + { id: 'es-containerhost', type: 'elasticsearch', preset: 'throughput' }, + { id: 'remote-es', type: 'remote_elasticsearch', preset: 'scale' }, ], }), }, @@ -52,9 +57,32 @@ describe('agents_per_output', () => { it('should return agent count by output type', async () => { const res = await getAgentsPerOutput(soClientMock, {} as unknown as ElasticsearchClient); expect(res).toEqual([ - { output_type: 'elasticsearch', count_as_data: 4, count_as_monitoring: 4 }, + { + output_type: 'elasticsearch', + count_as_data: 5, + count_as_monitoring: 5, + preset_counts: { + custom: 1, + balanced: 2, + throughput: 1, + scale: 0, + latency: 0, + }, + }, { output_type: 'logstash', count_as_data: 1, count_as_monitoring: 0 }, { output_type: 'kafka', count_as_data: 0, count_as_monitoring: 1 }, + { + output_type: 'remote_elasticsearch', + count_as_data: 1, + count_as_monitoring: 1, + preset_counts: { + custom: 0, + balanced: 0, + throughput: 0, + scale: 1, + latency: 0, + }, + }, ]); }); }); diff --git a/x-pack/plugins/fleet/server/collectors/agents_per_output.ts b/x-pack/plugins/fleet/server/collectors/agents_per_output.ts index 3ad09bcb51177..b7ed480fc61fc 100644 --- a/x-pack/plugins/fleet/server/collectors/agents_per_output.ts +++ b/x-pack/plugins/fleet/server/collectors/agents_per_output.ts @@ -6,8 +6,12 @@ */ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; + import _ from 'lodash'; +import { outputTypeSupportPresets } from '../../common/services/output_helpers'; +import type { AgentPolicy } from '../../common/types'; + import { SO_SEARCH_LIMIT } from '../../common'; import { agentPolicyService, outputService } from '../services'; @@ -15,6 +19,13 @@ export interface AgentsPerOutputType { output_type: string; count_as_data: number; count_as_monitoring: number; + preset_counts?: { + balanced: number; + custom: number; + latency: number; + scale: number; + throughput: number; + }; } export async function getAgentsPerOutput( @@ -28,38 +39,69 @@ export async function getAgentsPerOutput( outputs.find((output) => output.is_default_monitoring)?.id || ''; const outputsById = _.keyBy(outputs, 'id'); - const getOutputTypeById = (outputId: string): string => outputsById[outputId]?.type ?? ''; + const getDataOutputForAgentPolicy = (agentPolicy: AgentPolicy) => + outputsById[agentPolicy.data_output_id || defaultOutputId]; + const getMonitoringOutputForAgentPolicy = (agentPolicy: AgentPolicy) => + outputsById[agentPolicy.monitoring_output_id || defaultMonitoringOutputId]; - const { items } = await agentPolicyService.list(soClient, { + const { items: agentPolicies } = await agentPolicyService.list(soClient, { esClient, withAgentCount: true, page: 1, perPage: SO_SEARCH_LIMIT, }); + const outputTypes: { [key: string]: AgentsPerOutputType } = {}; - items - .filter((item) => (item.agents ?? 0) > 0) - .forEach((item) => { - const dataOutputType = getOutputTypeById(item.data_output_id || defaultOutputId); - if (!outputTypes[dataOutputType]) { - outputTypes[dataOutputType] = { - output_type: dataOutputType, + + agentPolicies + .filter((agentPolicy) => (agentPolicy.agents ?? 0) > 0) + .forEach((agentPolicy) => { + const dataOutput = getDataOutputForAgentPolicy(agentPolicy); + const monitoringOutput = getMonitoringOutputForAgentPolicy(agentPolicy); + + if (!outputTypes[dataOutput.type]) { + outputTypes[dataOutput.type] = { + output_type: dataOutput.type, count_as_data: 0, count_as_monitoring: 0, }; } - outputTypes[dataOutputType].count_as_data += item.agents ?? 0; - const monitoringOutputType = getOutputTypeById( - item.monitoring_output_id || defaultMonitoringOutputId - ); - if (!outputTypes[monitoringOutputType]) { - outputTypes[monitoringOutputType] = { - output_type: monitoringOutputType, + + outputTypes[dataOutput.type].count_as_data += agentPolicy.agents ?? 0; + + if (!outputTypes[monitoringOutput.type]) { + outputTypes[monitoringOutput.type] = { + output_type: monitoringOutput.type, count_as_data: 0, count_as_monitoring: 0, }; } - outputTypes[monitoringOutputType].count_as_monitoring += item.agents ?? 0; + outputTypes[monitoringOutput.type].count_as_monitoring += agentPolicy.agents ?? 0; }); + + outputs.forEach((output) => { + if (!outputTypeSupportPresets(output.type)) { + return; + } + + const outputTelemetryRecord = outputTypes[output.type]; + + if (!outputTelemetryRecord.preset_counts) { + outputTelemetryRecord.preset_counts = { + balanced: 0, + custom: 0, + latency: 0, + scale: 0, + throughput: 0, + }; + } + + if (output.preset && output.preset in outputTelemetryRecord.preset_counts) { + outputTelemetryRecord.preset_counts[ + output.preset as keyof typeof outputTelemetryRecord.preset_counts + ] += 1; + } + }); + return Object.values(outputTypes); } diff --git a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts index e2e7e9f7887e6..19eb7aa750658 100644 --- a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts +++ b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts @@ -221,6 +221,20 @@ describe('fleet usage telemetry', () => { active: true, policy_id: 'policy2', }, + { + create: { + _id: 'agent4', + }, + }, + { + agent: { + version: '8.6.0', + }, + last_checkin_status: 'online', + last_checkin: new Date(Date.now() - 1000 * 60 * 6).toISOString(), + active: true, + policy_id: 'policy3', + }, ], refresh: 'wait_for', }); @@ -361,6 +375,21 @@ describe('fleet usage telemetry', () => { }, { id: 'output3' } ); + await soClient.create( + 'ingest-outputs', + { + name: 'output4', + type: 'elasticsearch', + hosts: ['http://localhost:9200'], + is_default: false, + is_default_monitoring: false, + config_yaml: '', + ca_trusted_fingerprint: '', + proxy_id: null, + preset: 'balanced', + }, + { id: 'output4' } + ); await soClient.create( 'ingest-agent-policies', @@ -380,6 +409,24 @@ describe('fleet usage telemetry', () => { }, { id: 'policy2' } ); + await soClient.create( + 'ingest-agent-policies', + { + namespace: 'default', + monitoring_enabled: ['logs', 'metrics'], + name: 'Yet another policy', + description: 'Policy 3', + inactivity_timeout: 1209600, + status: 'active', + is_managed: false, + revision: 2, + updated_by: 'system', + schema_version: '1.0.0', + data_output_id: 'output4', + monitoring_output_id: 'output4', + }, + { id: 'policy3' } + ); }); afterAll(async () => { @@ -397,13 +444,13 @@ describe('fleet usage telemetry', () => { expect.objectContaining({ agents_enabled: true, agents: { - total_enrolled: 3, + total_enrolled: 4, healthy: 0, unhealthy: 0, inactive: 0, unenrolled: 1, - offline: 3, - total_all_statuses: 4, + offline: 4, + total_all_statuses: 5, updating: 0, }, fleet_server: { @@ -419,10 +466,10 @@ describe('fleet usage telemetry', () => { agents_per_version: [ { version: '8.6.0', - count: 2, + count: 3, healthy: 0, inactive: 0, - offline: 2, + offline: 3, unenrolled: 0, unhealthy: 0, updating: 0, @@ -439,7 +486,7 @@ describe('fleet usage telemetry', () => { }, ], agent_checkin_status: { error: 1, degraded: 1 }, - agents_per_policy: [2, 1], + agents_per_policy: [2, 1, 1], agents_per_os: [ { name: 'Ubuntu', @@ -463,6 +510,18 @@ describe('fleet usage telemetry', () => { count_as_monitoring: 1, output_type: 'logstash', }, + { + count_as_data: 1, + count_as_monitoring: 1, + output_type: 'elasticsearch', + preset_counts: { + balanced: 2, + custom: 0, + latency: 0, + scale: 0, + throughput: 0, + }, + }, ], fleet_server_config: { policies: [ @@ -477,7 +536,7 @@ describe('fleet usage telemetry', () => { ], }, agent_policies: { - count: 2, + count: 3, output_types: expect.arrayContaining(['elasticsearch', 'logstash', 'third_type']), }, agent_logs_panics_last_hour: [ diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts index e59de684264bf..fa7d2d769b900 100644 --- a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts @@ -76,6 +76,49 @@ export const fleetAgentsSchema: RootSchema = { description: 'Output type used by agent', }, }, + presets_counts: { + _meta: { + description: 'Count of agents per preset', + }, + properties: { + balanced: { + type: 'long', + _meta: { + description: 'Number of agents enrolled whose output uses the balanced preset', + }, + }, + custom: { + type: 'long', + _meta: { + description: 'Number of agents enrolled whose outputs uses the custom preset', + }, + }, + throughput: { + type: 'long', + _meta: { + description: 'Number of agents enrolled whose output uses the throughput preset', + }, + }, + scale: { + type: 'long', + _meta: { + description: 'Number of agents enrolled whose output uses the scale preset', + }, + }, + latency: { + type: 'long', + _meta: { + description: 'Number of agents enrolled whose output uses the latency preset', + }, + }, + }, + }, + output_preset: { + type: 'keyword', + _meta: { + description: 'Output preset used by agent, if applicable', + }, + }, count_as_data: { type: 'long', _meta: {