From ea57fb09c0969d8006b035ec5958134bf13124f0 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 9 Sep 2024 09:45:12 -0400 Subject: [PATCH] [Fleet] Introduce Async deploy policies (#191839) --- .../fleet/common/experimental_features.ts | 1 + .../services/agent_and_policies_count.tsx | 1 + .../create_agent_policies.ts | 66 +- .../scripts/create_agent_policies/fixtures.ts | 800 ++++++++++++++++++ x-pack/plugins/fleet/server/plugin.ts | 3 + .../deploy_agent_policies_task.ts | 91 ++ .../fleet/server/services/agent_policy.ts | 97 ++- ...et_server_policies_enrollment_keys.test.ts | 3 + .../fleet_server_policies_enrollment_keys.ts | 27 +- .../apis/fleet_proxies/crud.ts | 46 +- .../check_registered_task_types.ts | 1 + 11 files changed, 1078 insertions(+), 58 deletions(-) create mode 100644 x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts create mode 100644 x-pack/plugins/fleet/server/services/agent_policies/deploy_agent_policies_task.ts diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts index fb5ef4ca3ae8d..522af87e56015 100644 --- a/x-pack/plugins/fleet/common/experimental_features.ts +++ b/x-pack/plugins/fleet/common/experimental_features.ts @@ -27,6 +27,7 @@ const _allowedExperimentalValues = { advancedPolicySettings: true, useSpaceAwareness: false, enableReusableIntegrationPolicies: true, + asyncDeployPolicies: true, }; /** diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx index 9ea3db4e14851..e91002c12ad0c 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx @@ -25,6 +25,7 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { kuery: agentPolicyKuery, page: 1, perPage: SO_SEARCH_LIMIT, + noAgentCount: true, }); if (agentPolicies.error) { diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts b/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts index 83838b3443e90..db47c056ce12d 100644 --- a/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts +++ b/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts @@ -7,9 +7,13 @@ import { ToolingLog } from '@kbn/tooling-log'; import yargs from 'yargs'; +import { chunk } from 'lodash'; +import { LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../common/constants'; import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common'; +import { packagePolicyFixture } from './fixtures'; + const logger = new ToolingLog({ level: 'info', writeTo: process.stdout, @@ -26,13 +30,17 @@ const printUsage = () => const INDEX_BULK_OP = '{ "index":{ "_id": "{{id}}" } }\n'; -async function createAgentPoliciesDocsBulk(size: number) { +function getPolicyId(idx: number | string) { + return `test-policy-${idx}`; +} + +async function createAgentPoliciesDocsBulk(range: number[]) { const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64'); - const body = Array.from({ length: size }, (_, index) => index + 1) + const body = range .flatMap((idx) => [ INDEX_BULK_OP.replace( /{{id}}/, - `${LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE}:test-policy-${idx}` + `${LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE}:${getPolicyId(idx)}` ), JSON.stringify({ [LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE]: { @@ -79,9 +87,9 @@ async function createAgentPoliciesDocsBulk(size: number) { return data; } -async function createEnrollmentToken(size: number) { +async function createEnrollmentToken(range: number[]) { const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64'); - const body = Array.from({ length: size }, (_, index) => index + 1) + const body = range .flatMap((idx) => [ INDEX_BULK_OP.replace(/{{id}}/, `test-enrollment-token-${idx}`), JSON.stringify({ @@ -89,7 +97,7 @@ async function createEnrollmentToken(size: number) { api_key_id: 'faketest123', api_key: 'test==', name: `Test Policy ${idx}`, - policy_id: `${LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE}:test-policy-${idx}`, + policy_id: `${getPolicyId(idx)}`, namespaces: [], created_at: new Date().toISOString(), }) + '\n', @@ -113,6 +121,41 @@ async function createEnrollmentToken(size: number) { return data; } +async function createPackagePolicies(range: number[]) { + const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64'); + const body = range + .flatMap((idx) => [ + INDEX_BULK_OP.replace( + /{{id}}/, + `${LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE}:test-policy-${idx}` + ), + JSON.stringify( + packagePolicyFixture({ + idx, + agentPolicyId: getPolicyId(idx), + }) + ) + '\n', + ]) + .join(''); + + const res = await fetch(`${ES_URL}/.kibana_ingest/_bulk`, { + method: 'post', + body, + headers: { + Authorization: auth, + 'Content-Type': 'application/x-ndjson', + }, + }); + + const data = await res.json(); + + if (!data.items) { + logger.error('Error creating agent policies docs: ' + JSON.stringify(data)); + process.exit(1); + } + return data; +} + export async function run() { const { size: sizeArg = 500, @@ -129,6 +172,15 @@ export async function run() { const size = Number(sizeArg).valueOf(); logger.info(`Creating ${size} policies`); - await Promise.all([createAgentPoliciesDocsBulk(size), createEnrollmentToken(size)]); + + const range = Array.from({ length: size }, (_ignore, index) => index + 1); + + for (const rangePart of chunk(range, 200)) { + await Promise.all([ + createAgentPoliciesDocsBulk(rangePart), + createEnrollmentToken(rangePart), + createPackagePolicies(rangePart), + ]); + } logger.info(`Succesfuly created ${size} policies`); } diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts b/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts new file mode 100644 index 0000000000000..b10f412ac43fe --- /dev/null +++ b/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts @@ -0,0 +1,800 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const packagePolicyFixture = ({ + agentPolicyId, + idx, +}: { + idx: number; + agentPolicyId: string; +}) => ({ + 'ingest-package-policies': { + name: `system-test-${idx}`, + namespace: '', + description: '', + package: { + name: 'system', + title: 'System', + version: '1.60.3', + requires_root: true, + }, + enabled: true, + policy_id: agentPolicyId, + policy_ids: [agentPolicyId], + inputs: [ + { + type: 'logfile', + policy_template: 'system', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { + type: 'logs', + dataset: 'system.auth', + }, + vars: { + ignore_older: { + value: '72h', + type: 'text', + }, + paths: { + value: ['/var/log/auth.log*', '/var/log/secure*'], + type: 'text', + }, + preserve_original_event: { + value: false, + type: 'bool', + }, + tags: { + value: ['system-auth'], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'logfile-system.auth-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + ignore_older: '72h', + paths: ['/var/log/auth.log*', '/var/log/secure*'], + exclude_files: [`\.gz$`], + multiline: { + pattern: `^\s`, + match: 'after', + }, + tags: ['system-auth'], + processors: [ + { + add_locale: null, + }, + { + rename: { + fields: [ + { + from: 'message', + to: 'event.original', + }, + ], + ignore_missing: true, + fail_on_error: false, + }, + }, + { + syslog: { + field: 'event.original', + ignore_missing: true, + ignore_failure: true, + }, + }, + ], + }, + }, + { + enabled: true, + data_stream: { + type: 'logs', + dataset: 'system.syslog', + }, + vars: { + paths: { + value: ['/var/log/messages*', '/var/log/syslog*', '/var/log/system*'], + type: 'text', + }, + preserve_original_event: { + value: false, + type: 'bool', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + ignore_older: { + value: '72h', + type: 'text', + }, + exclude_files: { + value: [`\.gz$`], + type: 'text', + }, + }, + id: 'logfile-system.syslog-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + paths: ['/var/log/messages*', '/var/log/syslog*', '/var/log/system*'], + exclude_files: [`\.gz$`], + multiline: { + pattern: `^\s`, + match: 'after', + }, + processors: [ + { + add_locale: null, + }, + ], + tags: null, + ignore_older: '72h', + }, + }, + ], + }, + { + type: 'winlog', + policy_template: 'system', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { + type: 'logs', + dataset: 'system.application', + }, + vars: { + preserve_original_event: { + value: false, + type: 'bool', + }, + event_id: { + type: 'text', + }, + ignore_older: { + value: '72h', + type: 'text', + }, + language: { + value: 0, + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + custom: { + value: `# Winlog configuration example +#batch_read_size: 100`, + type: 'yaml', + }, + }, + id: 'winlog-system.application-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + name: 'Application', + condition: "${host.platform} == 'windows'", + ignore_older: '72h', + }, + }, + { + enabled: true, + data_stream: { + type: 'logs', + dataset: 'system.security', + }, + vars: { + preserve_original_event: { + value: false, + type: 'bool', + }, + event_id: { + type: 'text', + }, + ignore_older: { + value: '72h', + type: 'text', + }, + language: { + value: 0, + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + custom: { + value: `# Winlog configuration example +#batch_read_size: 100`, + type: 'yaml', + }, + }, + id: 'winlog-system.security-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + name: 'Security', + condition: "${host.platform} == 'windows'", + ignore_older: '72h', + }, + }, + { + enabled: true, + data_stream: { + type: 'logs', + dataset: 'system.system', + }, + vars: { + preserve_original_event: { + value: false, + type: 'bool', + }, + event_id: { + type: 'text', + }, + ignore_older: { + value: '72h', + type: 'text', + }, + language: { + value: 0, + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + custom: { + value: `# Winlog configuration example +#batch_read_size: 100`, + type: 'yaml', + }, + }, + id: 'winlog-system.system-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + name: 'System', + condition: "${host.platform} == 'windows'", + ignore_older: '72h', + }, + }, + ], + }, + { + type: 'system/metrics', + policy_template: 'system', + enabled: true, + streams: [ + { + enabled: false, + data_stream: { + type: 'metrics', + dataset: 'system.core', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + 'core.metrics': { + value: ['percentages'], + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.core-1f70ab3b-5631-4239-9f87-2881e3986a0a', + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.cpu', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + 'cpu.metrics': { + value: ['percentages', 'normalized_percentages'], + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.cpu-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['cpu'], + 'cpu.metrics': ['percentages', 'normalized_percentages'], + period: '10s', + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.diskio', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + 'diskio.include_devices': { + value: [], + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.diskio-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['diskio'], + 'diskio.include_devices': null, + period: '10s', + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.filesystem', + }, + vars: { + period: { + value: '1m', + type: 'text', + }, + 'filesystem.ignore_types': { + value: [], + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + value: `- drop_event.when.regexp: + system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)`, + type: 'yaml', + }, + }, + id: 'system/metrics-system.filesystem-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['filesystem'], + period: '1m', + processors: [ + { + 'drop_event.when.regexp': { + 'system.filesystem.mount_point': + '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)', + }, + }, + ], + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.fsstat', + }, + vars: { + period: { + value: '1m', + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + value: `- drop_event.when.regexp: + system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)`, + type: 'yaml', + }, + }, + id: 'system/metrics-system.fsstat-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['fsstat'], + period: '1m', + processors: [ + { + 'drop_event.when.regexp': { + 'system.fsstat.mount_point': '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)', + }, + }, + ], + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.load', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.load-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['load'], + condition: "${host.platform} != 'windows'", + period: '10s', + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.memory', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.memory-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['memory'], + period: '10s', + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.network', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + 'network.interfaces': { + value: [], + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.network-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['network'], + period: '10s', + 'network.interfaces': null, + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.process', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + 'process.include_top_n.by_cpu': { + value: 5, + type: 'integer', + }, + 'process.include_top_n.by_memory': { + value: 5, + type: 'integer', + }, + 'process.cmdline.cache.enabled': { + value: true, + type: 'bool', + }, + 'process.cgroups.enabled': { + value: false, + type: 'bool', + }, + 'process.env.whitelist': { + value: [], + type: 'text', + }, + 'process.include_cpu_ticks': { + value: false, + type: 'bool', + }, + processes: { + value: ['.*'], + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.process-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['process'], + period: '10s', + 'process.include_top_n.by_cpu': 5, + 'process.include_top_n.by_memory': 5, + 'process.cmdline.cache.enabled': true, + 'process.cgroups.enabled': false, + 'process.include_cpu_ticks': false, + processes: ['.*'], + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.process.summary', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.process.summary-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['process_summary'], + period: '10s', + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.socket_summary', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.socket_summary-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['socket_summary'], + period: '10s', + }, + }, + { + enabled: true, + data_stream: { + type: 'metrics', + dataset: 'system.uptime', + }, + vars: { + period: { + value: '10s', + type: 'text', + }, + tags: { + value: [], + type: 'text', + }, + processors: { + type: 'yaml', + }, + }, + id: 'system/metrics-system.uptime-1f70ab3b-5631-4239-9f87-2881e3986a0a', + compiled_stream: { + metricsets: ['uptime'], + period: '10s', + }, + }, + ], + vars: { + 'system.hostfs': { + type: 'text', + }, + }, + }, + { + type: 'httpjson', + policy_template: 'system', + enabled: false, + streams: [ + { + enabled: false, + data_stream: { + type: 'logs', + dataset: 'system.application', + }, + vars: { + interval: { + value: '10s', + type: 'text', + }, + search: { + value: 'search sourcetype="XmlWinEventLog:Application"', + type: 'text', + }, + tags: { + value: ['forwarded'], + type: 'text', + }, + }, + id: 'httpjson-system.application-1f70ab3b-5631-4239-9f87-2881e3986a0a', + }, + { + enabled: false, + data_stream: { + type: 'logs', + dataset: 'system.security', + }, + vars: { + interval: { + value: '10s', + type: 'text', + }, + search: { + value: 'search sourcetype="XmlWinEventLog:Security"', + type: 'text', + }, + tags: { + value: ['forwarded'], + type: 'text', + }, + }, + id: 'httpjson-system.security-1f70ab3b-5631-4239-9f87-2881e3986a0a', + }, + { + enabled: false, + data_stream: { + type: 'logs', + dataset: 'system.system', + }, + vars: { + interval: { + value: '10s', + type: 'text', + }, + search: { + value: 'search sourcetype="XmlWinEventLog:System"', + type: 'text', + }, + tags: { + value: ['forwarded'], + type: 'text', + }, + }, + id: 'httpjson-system.system-1f70ab3b-5631-4239-9f87-2881e3986a0a', + }, + ], + vars: { + url: { + value: 'https://server.example.com:8089', + type: 'text', + }, + enable_request_tracer: { + type: 'bool', + }, + username: { + type: 'text', + }, + password: { + type: 'password', + }, + token: { + type: 'password', + }, + preserve_original_event: { + value: false, + type: 'bool', + }, + ssl: { + value: `#certificate_authorities: +# - | +# -----BEGIN CERTIFICATE----- +# MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF +# ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 +# MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB +# BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n +# fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl +# 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t +# /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP +# PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41 +# CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O +# BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux +# 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D +# 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw +# 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA +# H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu +# 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0 +# yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk +# sxSmbIUfc2SGJGCJD4I= +# -----END CERTIFICATE----- +`, + type: 'yaml', + }, + }, + }, + ], + output_id: null, + revision: 1, + created_at: '2024-08-30T13:45:51.197Z', + created_by: 'system', + updated_at: '2024-08-30T13:45:51.197Z', + updated_by: 'system', + }, + type: 'ingest-package-policies', + references: [], + managed: false, + coreMigrationVersion: '8.8.0', + typeMigrationVersion: '10.14.0', + updated_at: '2024-08-30T13:45:51.197Z', + created_at: '2024-08-30T13:45:51.197Z', +}); diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 8b472b8b4dfbb..c2451005ecb4a 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -131,6 +131,7 @@ import { FleetMetricsTask } from './services/metrics/fleet_metrics_task'; import { fetchAgentMetrics } from './services/metrics/fetch_agent_metrics'; import { registerIntegrationFieldsExtractor } from './services/register_integration_fields_extractor'; import { registerUpgradeManagedPackagePoliciesTask } from './services/setup/managed_package_policies'; +import { registerDeployAgentPoliciesTask } from './services/agent_policies/deploy_agent_policies_task'; export interface FleetSetupDeps { security: SecurityPluginSetup; @@ -600,6 +601,8 @@ export class FleetPlugin this.telemetryEventsSender.setup(deps.telemetry); // Register task registerUpgradeManagedPackagePoliciesTask(deps.taskManager); + registerDeployAgentPoliciesTask(deps.taskManager); + this.bulkActionsResolver = new BulkActionsResolver(deps.taskManager, core); this.checkDeletedFilesTask = new CheckDeletedFilesTask({ core, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/deploy_agent_policies_task.ts b/x-pack/plugins/fleet/server/services/agent_policies/deploy_agent_policies_task.ts new file mode 100644 index 0000000000000..1dc1b4a1d7a79 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/agent_policies/deploy_agent_policies_task.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import { v4 as uuidv4 } from 'uuid'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; + +import { agentPolicyService, appContextService } from '..'; +import { runWithCache } from '../epm/packages/cache'; + +const TASK_TYPE = 'fleet:deploy_agent_policies'; + +export function registerDeployAgentPoliciesTask(taskManagerSetup: TaskManagerSetupContract) { + taskManagerSetup.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Fleet Deploy policies', + timeout: '5m', + maxAttempts: 3, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + const agentPolicyIdsWithSpace: Array<{ id: string; spaceId?: string }> = + taskInstance.params.agentPolicyIdsWithSpace; + let cancelled = false; + return { + async run() { + if (!agentPolicyIdsWithSpace.length) { + return; + } + appContextService + .getLogger() + .debug(`Deploying ${agentPolicyIdsWithSpace.length} policies`); + const agentPoliciesIdsIndexedBySpace = agentPolicyIdsWithSpace.reduce( + (acc, { id, spaceId = DEFAULT_SPACE_ID }) => { + if (!acc[spaceId]) { + acc[spaceId] = []; + } + + acc[spaceId].push(id); + + return acc; + }, + {} as { [k: string]: string[] } + ); + + await runWithCache(async () => { + for (const [spaceId, agentPolicyIds] of Object.entries( + agentPoliciesIdsIndexedBySpace + )) { + if (cancelled) { + throw new Error('Task has been cancelled'); + } + await agentPolicyService.deployPolicies( + appContextService.getInternalUserSOClientForSpaceId(spaceId), + agentPolicyIds + ); + } + }); + }, + async cancel() { + cancelled = true; + }, + }; + }, + }, + }); +} + +export async function scheduleDeployAgentPoliciesTask( + taskManagerStart: TaskManagerStartContract, + agentPolicyIdsWithSpace: Array<{ id: string; spaceId?: string }> +) { + if (!agentPolicyIdsWithSpace.length) { + return; + } + + await taskManagerStart.ensureScheduled({ + id: `${TASK_TYPE}:${uuidv4()}`, + scope: ['fleet'], + params: { agentPolicyIdsWithSpace }, + taskType: TASK_TYPE, + runAt: new Date(Date.now() + 3 * 1000), + state: {}, + }); +} diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index b5f885b6743bf..38c785520b510 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -112,6 +112,7 @@ import { createSoFindIterable } from './utils/create_so_find_iterable'; import { isAgentlessEnabled } from './utils/agentless'; import { validatePolicyNamespaceForSpace } from './spaces/policy_namespaces'; import { isSpaceAwarenessEnabled } from './spaces/helpers'; +import { scheduleDeployAgentPoliciesTask } from './agent_policies/deploy_agent_policies_task'; const KEY_EDITABLE_FOR_MANAGED_POLICIES = ['namespace']; @@ -585,29 +586,36 @@ class AgentPolicyService { } } - const agentPolicies = await pMap( - agentPoliciesSO.saved_objects, - async (agentPolicySO) => { - const agentPolicy = mapAgentPolicySavedObjectToAgentPolicy(agentPolicySO); - if (withPackagePolicies) { - agentPolicy.package_policies = - (await packagePolicyService.findAllForAgentPolicy(soClient, agentPolicySO.id)) || []; - } - if (options.withAgentCount) { - await getAgentsByKuery(appContextService.getInternalUserESClient(), soClient, { - showInactive: true, - perPage: 0, - page: 1, - kuery: `${AGENTS_PREFIX}.policy_id:${agentPolicy.id}`, - }).then(({ total }) => (agentPolicy.agents = total)); - } else { - agentPolicy.agents = 0; - } + const agentPolicies = agentPoliciesSO.saved_objects.map((agentPolicySO) => { + const agentPolicy = mapAgentPolicySavedObjectToAgentPolicy(agentPolicySO); + agentPolicy.agents = 0; + return agentPolicy; + }); - return agentPolicy; - }, - { concurrency: 50 } - ); + if (options.withAgentCount || withPackagePolicies) { + await pMap( + agentPolicies, + async (agentPolicy) => { + if (withPackagePolicies) { + agentPolicy.package_policies = + (await packagePolicyService.findAllForAgentPolicy(soClient, agentPolicy.id)) || []; + } + if (options.withAgentCount) { + await getAgentsByKuery(appContextService.getInternalUserESClient(), soClient, { + showInactive: true, + perPage: 0, + page: 1, + kuery: `${AGENTS_PREFIX}.policy_id:${agentPolicy.id}`, + }).then(({ total }) => (agentPolicy.agents = total)); + } else { + agentPolicy.agents = 0; + } + + return agentPolicy; + }, + { concurrency: 50 } + ); + } for (const agentPolicy of agentPolicies) { auditLoggingService.writeCustomSoAuditLog({ @@ -972,14 +980,24 @@ class AgentPolicyService { } ); - await pMap( - savedObjectsResults, - (policy) => - this.triggerAgentPolicyUpdatedEvent(esClient, 'updated', policy.id, { + if (appContextService.getExperimentalFeatures().asyncDeployPolicies) { + await scheduleDeployAgentPoliciesTask( + appContextService.getTaskManagerStart()!, + savedObjectsResults.map((policy) => ({ + id: policy.id, spaceId: policy.namespaces?.[0], - }), - { concurrency: 50 } - ); + })) + ); + } else { + await pMap( + savedObjectsResults, + (policy) => + this.triggerAgentPolicyUpdatedEvent(esClient, 'updated', policy.id, { + spaceId: policy.namespaces?.[0], + }), + { concurrency: 50 } + ); + } return res; } @@ -1221,7 +1239,7 @@ class AgentPolicyService { agentPolicy: agentPolicies?.find((policy) => policy.id === agentPolicyId), }), { - concurrency: 50, + concurrency: 20, } ); @@ -1593,10 +1611,21 @@ class AgentPolicyService { const config = appContextService.getConfig(); const batchSize = config?.setup?.agentPolicySchemaUpgradeBatchSize ?? 100; const policyIds = updatedPoliciesSuccess.map((policy) => policy.id); - await asyncForEach( - chunk(policyIds, batchSize), - async (policyIdsBatch) => await this.deployPolicies(soClient, policyIdsBatch) - ); + + if (appContextService.getExperimentalFeatures().asyncDeployPolicies) { + await scheduleDeployAgentPoliciesTask( + appContextService.getTaskManagerStart()!, + updatedPoliciesSuccess.map((policy) => ({ + id: policy.id, + spaceId: policy.namespaces?.[0], + })) + ); + } else { + await asyncForEach( + chunk(policyIds, batchSize), + async (policyIdsBatch) => await this.deployPolicies(soClient, policyIdsBatch) + ); + } return { updatedPolicies: updatedPoliciesSuccess, failedPolicies }; } diff --git a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts index 3a90fb5a90fa3..07ec6593ec9e5 100644 --- a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts +++ b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts @@ -29,6 +29,9 @@ describe('ensureAgentPoliciesFleetServerKeysAndPolicies', () => { jest.mocked(appContextService).getSecurity.mockReturnValue({ authc: { apiKeys: { areAPIKeysEnabled: async () => true } }, } as any); + jest.mocked(appContextService).getExperimentalFeatures.mockReturnValue({ + asyncDeployPolicies: false, + } as any); mockedEnsureDefaultEnrollmentAPIKeyForAgentPolicy.mockReset(); mockedAgentPolicyService.getLatestFleetPolicy.mockReset(); diff --git a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts index 07f368c3e7400..f5ed816d96e61 100644 --- a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts +++ b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts @@ -12,6 +12,7 @@ import { agentPolicyService } from '../agent_policy'; import { ensureDefaultEnrollmentAPIKeyForAgentPolicy } from '../api_keys'; import { SO_SEARCH_LIMIT } from '../../constants'; import { appContextService } from '../app_context'; +import { scheduleDeployAgentPoliciesTask } from '../agent_policies/deploy_agent_policies_task'; export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ logger, @@ -35,7 +36,7 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ perPage: SO_SEARCH_LIMIT, }); - const outdatedAgentPolicyIds: string[] = []; + const outdatedAgentPolicyIds: Array<{ id: string; spaceId?: string }> = []; await pMap( agentPolicies, @@ -46,7 +47,7 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ ]); if ((latestFleetPolicy?.revision_idx ?? -1) < agentPolicy.revision) { - outdatedAgentPolicyIds.push(agentPolicy.id); + outdatedAgentPolicyIds.push({ id: agentPolicy.id, spaceId: agentPolicy.space_ids?.[0] }); } }, { @@ -54,9 +55,23 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ } ); - if (outdatedAgentPolicyIds.length) { - await agentPolicyService.deployPolicies(soClient, outdatedAgentPolicyIds).catch((error) => { - logger.warn(`Error deploying policies: ${error.message}`, { error }); - }); + if (!outdatedAgentPolicyIds.length) { + return; + } + + if (appContextService.getExperimentalFeatures().asyncDeployPolicies) { + return scheduleDeployAgentPoliciesTask( + appContextService.getTaskManagerStart()!, + outdatedAgentPolicyIds + ); + } else { + return agentPolicyService + .deployPolicies( + soClient, + outdatedAgentPolicyIds.map(({ id }) => id) + ) + .catch((error) => { + logger.warn(`Error deploying policies: ${error.message}`, { error }); + }); } } diff --git a/x-pack/test/fleet_api_integration/apis/fleet_proxies/crud.ts b/x-pack/test/fleet_api_integration/apis/fleet_proxies/crud.ts index d8ad522209bb8..085652d2ddc95 100644 --- a/x-pack/test/fleet_api_integration/apis/fleet_proxies/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/fleet_proxies/crud.ts @@ -6,6 +6,8 @@ */ import expect from '@kbn/expect'; +import pRetry from 'p-retry'; + import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; @@ -145,6 +147,7 @@ export default function (providerContext: FtrProviderContext) { describe('PUT /proxies/{itemId}', () => { it('should allow to update an existing fleet proxy', async function () { + const fleetPolicyBefore = await getLatestFleetPolicies(policyId); await supertest .put(`/api/fleet/proxies/${existingId}`) .set('kbn-xsrf', 'xxxx') @@ -160,13 +163,23 @@ export default function (providerContext: FtrProviderContext) { expect(fleetServerHost.name).to.eql('Test 123 updated'); - const fleetPolicyAfter = await getLatestFleetPolicies(policyId); - expect(fleetPolicyAfter?.data?.fleet?.proxy_url).to.be('https://testupdated.fr:3232'); - expect(fleetPolicyAfter?.data?.outputs?.[outputId].proxy_url).to.be( - 'https://testupdated.fr:3232' - ); - expect(fleetPolicyAfter?.data?.agent.download.proxy_url).to.be( - 'https://testupdated.fr:3232' + await pRetry( + async () => { + const fleetPolicyAfter = await getLatestFleetPolicies(policyId); + if (fleetPolicyAfter.revision_idx === fleetPolicyBefore.revision_idx) { + throw new Error('fleet server policy not deployed'); + } + expect(fleetPolicyAfter?.data?.fleet?.proxy_url).to.be('https://testupdated.fr:3232'); + expect(fleetPolicyAfter?.data?.outputs?.[outputId].proxy_url).to.be( + 'https://testupdated.fr:3232' + ); + expect(fleetPolicyAfter?.data?.agent.download.proxy_url).to.be( + 'https://testupdated.fr:3232' + ); + }, + { + maxRetryTime: 30 * 1000, // 30s for the task to run + } ); }); @@ -183,15 +196,26 @@ export default function (providerContext: FtrProviderContext) { describe('DELETE /proxies/{itemId}', () => { it('should allow to delete an existing fleet proxy', async function () { + const fleetPolicyBefore = await getLatestFleetPolicies(policyId); await supertest .delete(`/api/fleet/proxies/${existingId}`) .set('kbn-xsrf', 'xxxx') .expect(200); - const fleetPolicyAfter = await getLatestFleetPolicies(policyId); - expect(fleetPolicyAfter?.data?.fleet?.proxy_url).to.be(undefined); - expect(fleetPolicyAfter?.data?.outputs?.[outputId].proxy_url).to.be(undefined); - expect(fleetPolicyAfter?.data?.agent.download.proxy_url).to.be(undefined); + await pRetry( + async () => { + const fleetPolicyAfter = await getLatestFleetPolicies(policyId); + if (fleetPolicyAfter.revision_idx === fleetPolicyBefore.revision_idx) { + throw new Error('fleet server policy not deployed'); + } + expect(fleetPolicyAfter?.data?.fleet?.proxy_url).to.be(undefined); + expect(fleetPolicyAfter?.data?.outputs?.[outputId].proxy_url).to.be(undefined); + expect(fleetPolicyAfter?.data?.agent.download.proxy_url).to.be(undefined); + }, + { + maxRetryTime: 30 * 1000, // 30s for the task to run + } + ); }); }); }); diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index a488cd1a47171..cdc826919d0ab 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -138,6 +138,7 @@ export default function ({ getService }: FtrProviderContext) { 'endpoint:metadata-check-transforms-task', 'endpoint:user-artifact-packager', 'fleet:check-deleted-files-task', + 'fleet:deploy_agent_policies', 'fleet:reassign_action:retry', 'fleet:request_diagnostics:retry', 'fleet:setup:upgrade_managed_package_policies',