From a8a794f0d16c85623541b115058e58b16eb83506 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 6 Nov 2024 11:02:03 -0500 Subject: [PATCH 1/3] [Fleet] Improve input template API yaml comments --- .../redis_1_18_0_package_info.json | 18 ++++ .../redis_1_18_0_streams_template.ts | 9 ++ .../get_templates_inputs.test.ts.snap | 28 ++--- .../epm/packages/get_template_inputs.ts | 100 ++++++++++++------ 4 files changed, 113 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_package_info.json b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_package_info.json index 57c9b0c68fac9..5aa5c4a878baf 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_package_info.json +++ b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_package_info.json @@ -139,6 +139,15 @@ "show_user": false, "default": "20s" }, + { + "name": "tags", + "type": "text", + "title": "Tags", + "multi": true, + "required": false, + "show_user": false, + "default": "" + }, { "name": "maxconn", "type": "integer", @@ -203,6 +212,15 @@ { "input": "redis/metrics", "vars": [ + { + "name": "tags_streams", + "type": "text", + "title": "Tags in streams", + "multi": true, + "required": false, + "show_user": false, + "default": "" + }, { "name": "key.patterns", "type": "yaml", diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_streams_template.ts b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_streams_template.ts index 5ff46f358bbe7..207a48d7132a5 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_streams_template.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_streams_template.ts @@ -76,6 +76,15 @@ period: {{period}} processors: {{processors}} {{/if}} +tags: + - test +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +tags_streams: +{{#each tags_streams as |tag i|}} + - {{tag}} +{{/each}} `), ], ]); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap index b3a428c0e5a55..9a68a746b2bae 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap @@ -8,17 +8,16 @@ exports[`Fleet - getTemplateInputs should work for input package 1`] = ` streams: # Custom log file: Custom log file - id: logfile-log.logs - data_stream: - dataset: - # Dataset name: Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use \`-\` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). - - paths: - - # Log file path: Path to log files to be collected - exclude_files: - - # Exclude files: Patterns to be ignored ignore_older: 72h - tags: - - # Tags: Tags to include in the published event + # data_stream: + # dataset: + # # Dataset name: Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use \`-\` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). + # paths: + # - # Log file path: Path to log files to be collected + # exclude_files: + # - # Exclude files: Patterns to be ignored + # tags: + # - # Tags: Tags to include in the published event " `; @@ -49,8 +48,13 @@ exports[`Fleet - getTemplateInputs should work for integration package 1`] = ` pattern: '*' maxconn: 10 network: tcp - username: # Username - password: # Password period: 10s + tags: + - test + # - # Tags + # username: # Username + # password: # Password + # tags_streams: + # - # Tags in streams " `; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts index 8c63f4b093dd0..35391d6362648 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts @@ -268,21 +268,7 @@ function addCommentsToYaml( pkgInput.description ? `: ${pkgInput.description}` : '' }`; - yamlDoc.visit(inputItem, { - Scalar(key, node) { - if (node.value) { - const val = node.value.toString(); - for (const varDef of pkgInput.vars ?? []) { - const placeholder = getPlaceholder(varDef); - if (val.includes(placeholder)) { - node.comment = ` ${varDef.title}${ - varDef.description ? `: ${varDef.description}` : '' - }`; - } - } - } - }, - }); + commentVariablesInYaml(inputItem, pkgInput.vars ?? []); const yamlStreams = inputItem.get('streams'); if (!yamlDoc.isCollection(yamlStreams)) { @@ -300,21 +286,7 @@ function addCommentsToYaml( streamItem.commentBefore = ` ${pkgStream.title}${ pkgStream.description ? `: ${pkgStream.description}` : '' }`; - yamlDoc.visit(streamItem, { - Scalar(key, node) { - if (node.value) { - const val = node.value.toString(); - for (const varDef of pkgStream.vars ?? []) { - const placeholder = getPlaceholder(varDef); - if (val.includes(placeholder)) { - node.comment = ` ${varDef.title}${ - varDef.description ? `: ${varDef.description}` : '' - }`; - } - } - } - }, - }); + commentVariablesInYaml(streamItem, pkgStream.vars ?? []); } } }); @@ -324,3 +296,71 @@ function addCommentsToYaml( return doc.toString(); } + +function commentVariablesInYaml(rootNode: yamlDoc.Node, vars: RegistryVarsEntry[] = []) { + // Node need to be deleted after the end of the visit to be able to visit every node + const toDeleteFn: Array<() => void> = []; + yamlDoc.visit(rootNode, { + Scalar(key, node, path) { + if (node.value) { + const val = node.value.toString(); + for (const varDef of vars) { + const placeholder = getPlaceholder(varDef); + if (val.includes(placeholder)) { + node.comment = ` ${varDef.title}${varDef.description ? `: ${varDef.description}` : ''}`; + + const paths = [...path].reverse(); + + let prevPart: yamlDoc.Node | yamlDoc.Document | yamlDoc.Pair = node; + + for (const pathPart of paths) { + if (yamlDoc.isCollection(pathPart)) { + // If only one items in the collection comment the whole collection + if (pathPart.items.length === 1) { + continue; + } + } + if (yamlDoc.isSeq(pathPart)) { + const commentDoc = new yamlDoc.Document(new yamlDoc.YAMLSeq()); + commentDoc.add(prevPart); + const commentStr = commentDoc.toString().trimEnd(); + pathPart.comment = pathPart.comment + ? `${pathPart.comment} ${commentStr}` + : ` ${commentStr}`; + const keyToDelete = prevPart; + + toDeleteFn.push(() => { + pathPart.items.forEach((item, index) => { + if (item === keyToDelete) { + pathPart.delete(new yamlDoc.Scalar(index)); + } + }); + }); + return; + } + + if (yamlDoc.isMap(pathPart)) { + if (yamlDoc.isPair(prevPart)) { + const commentDoc = new yamlDoc.Document(new yamlDoc.YAMLMap()); + commentDoc.add(prevPart); + const commentStr = commentDoc.toString().trimEnd(); + + pathPart.comment = pathPart.comment + ? `${pathPart.comment}\n ${commentStr.toString()}` + : ` ${commentStr.toString()}`; + const keyToDelete = prevPart.key; + toDeleteFn.push(() => pathPart.delete(keyToDelete)); + } + return; + } + + prevPart = pathPart; + } + } + } + } + }, + }); + + toDeleteFn.forEach((deleteFn) => deleteFn()); +} From 59804d65f04bdb887c70be546faf5e05bb747c22 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 6 Nov 2024 14:57:41 -0500 Subject: [PATCH 2/3] fix dynamic stream id --- .../package_policies_to_agent_inputs.ts | 8 +++- .../get_templates_inputs.test.ts.snap | 21 ++++++++++ .../epm/packages/get_template_inputs.ts | 38 +++++++++++++++---- .../epm/packages/get_templates_inputs.test.ts | 13 +++++++ 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts index 807312fe5e7cb..f2a39c6f8800b 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts @@ -112,7 +112,8 @@ export const mergeInputsOverrides = ( export const getFullInputStreams = ( input: PackagePolicyInput, - allStreamEnabled: boolean = false + allStreamEnabled: boolean = false, + streamsOriginalIdsMap?: Map // Map of stream ids ): FullAgentPolicyInputStream => { return { ...(input.compiled_input || {}), @@ -121,8 +122,9 @@ export const getFullInputStreams = ( streams: input.streams .filter((stream) => stream.enabled || allStreamEnabled) .map((stream) => { + const streamId = stream.id; const fullStream: FullAgentPolicyInputStream = { - id: stream.id, + id: streamId, data_stream: stream.data_stream, ...stream.compiled_stream, ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { @@ -130,6 +132,8 @@ export const getFullInputStreams = ( return acc; }, {} as { [k: string]: any }), }; + streamsOriginalIdsMap?.set(fullStream.id, streamId); + return fullStream; }), } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap index 9a68a746b2bae..56e4f7ee09319 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap @@ -58,3 +58,24 @@ exports[`Fleet - getTemplateInputs should work for integration package 1`] = ` # - # Tags in streams " `; + +exports[`Fleet - getTemplateInputs should work for package with dynamic ids 1`] = ` +"inputs: + # Collect Docker container logs: Collecting docker container logs + - id: docker-filestream + type: filestream + streams: + # Collect Docker container logs: Collect Docker container logs + - id: docker-container-logs-\${docker.container.name}-\${docker.container.id} + data_stream: + dataset: docker.container_logs + type: logs + paths: + - /var/lib/docker/containers/\${docker.container.id}/*-json.log + parsers: + - container: + stream: all + format: docker + # condition: # Condition: Condition to filter when to apply this datastream. Refer to [Docker provider](https://www.elastic.co/guide/en/fleet/current/docker-provider.html) to find the available keys and to [Conditions](https://www.elastic.co/guide/en/fleet/current/dynamic-input-configuration.html#conditions) on how to use the available keys in conditions. +" +`; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts index 35391d6362648..f76b739904f3f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts @@ -51,20 +51,31 @@ type PackageWithInputAndStreamIndexed = Record< // Function based off storedPackagePolicyToAgentInputs, it only creates the `streams` section instead of the FullAgentPolicyInput export const templatePackagePolicyToFullInputStreams = ( - packagePolicyInputs: PackagePolicyInput[] + packagePolicyInputs: PackagePolicyInput[], + inputAndStreamsIdsMap?: Map }> ): TemplateAgentPolicyInput[] => { const fullInputsStreams: TemplateAgentPolicyInput[] = []; if (!packagePolicyInputs || packagePolicyInputs.length === 0) return fullInputsStreams; packagePolicyInputs.forEach((input) => { + const streamsIdsMap = new Map(); + + const inputId = input.policy_template + ? `${input.policy_template}-${input.type}` + : `${input.type}`; const fullInputStream = { // @ts-ignore-next-line the following id is actually one level above the one in fullInputStream, but the linter thinks it gets overwritten - id: input.policy_template ? `${input.policy_template}-${input.type}` : `${input.type}`, + id: inputId, type: input.type, - ...getFullInputStreams(input, true), + ...getFullInputStreams(input, true, streamsIdsMap), }; + inputAndStreamsIdsMap?.set(fullInputStream.id, { + originalId: inputId, + streams: streamsIdsMap, + }); + // deeply merge the input.config values with the full policy input stream merge( fullInputStream, @@ -167,8 +178,13 @@ export async function getTemplateInputs( ...emptyPackagePolicy, inputs: compiledInputs, }; + const inputIdsDestinationMap = new Map< + string, + { originalId: string; streams: Map } + >(); const inputs = templatePackagePolicyToFullInputStreams( - packagePolicyWithInputs.inputs as PackagePolicyInput[] + packagePolicyWithInputs.inputs as PackagePolicyInput[], + inputIdsDestinationMap ); if (format === 'json') { @@ -181,7 +197,7 @@ export async function getTemplateInputs( sortKeys: _sortYamlKeys, } ); - return addCommentsToYaml(yaml, buildIndexedPackage(packageInfo)); + return addCommentsToYaml(yaml, buildIndexedPackage(packageInfo), inputIdsDestinationMap); } return { inputs: [] }; @@ -247,7 +263,8 @@ function buildIndexedPackage(packageInfo: PackageInfo): PackageWithInputAndStrea function addCommentsToYaml( yaml: string, - packageIndexInputAndStreams: PackageWithInputAndStreamIndexed + packageIndexInputAndStreams: PackageWithInputAndStreamIndexed, + inputIdsDestinationMap: Map }> ) { const doc = yamlDoc.parseDocument(yaml); // Add input and streams comments @@ -261,7 +278,9 @@ function addCommentsToYaml( if (!yamlDoc.isScalar(inputIdNode)) { return; } - const inputId = inputIdNode.value as string; + const inputId = + inputIdsDestinationMap.get(inputIdNode.value as string)?.originalId ?? + (inputIdNode.value as string); const pkgInput = packageIndexInputAndStreams[inputId]; if (pkgInput) { inputItem.commentBefore = ` ${pkgInput.title}${ @@ -280,7 +299,10 @@ function addCommentsToYaml( } const streamIdNode = streamItem.get('id', true); if (yamlDoc.isScalar(streamIdNode)) { - const streamId = streamIdNode.value as string; + const streamId = + inputIdsDestinationMap + .get(inputIdNode.value as string) + ?.streams?.get(streamIdNode.value as string) ?? (streamIdNode.value as string); const pkgStream = pkgInput.streams[streamId]; if (pkgStream) { streamItem.commentBefore = ` ${pkgStream.title}${ diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_templates_inputs.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_templates_inputs.test.ts index 087002f212852..1a3738d8eaa82 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_templates_inputs.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_templates_inputs.test.ts @@ -16,6 +16,7 @@ import REDIS_1_18_0_PACKAGE_INFO from './__fixtures__/redis_1_18_0_package_info. import { getPackageAssetsMap, getPackageInfo } from './get'; import { REDIS_ASSETS_MAP } from './__fixtures__/redis_1_18_0_streams_template'; import { LOGS_2_3_0_ASSETS_MAP, LOGS_2_3_0_PACKAGE_INFO } from './__fixtures__/logs_2_3_0'; +import { DOCKER_2_11_0_PACKAGE_INFO, DOCKER_2_11_0_ASSETS_MAP } from './__fixtures__/docker_2_11_0'; jest.mock('./get'); @@ -41,6 +42,7 @@ packageInfoCache.set('limited_package-0.0.0', { packageInfoCache.set('redis-1.18.0', REDIS_1_18_0_PACKAGE_INFO); packageInfoCache.set('log-2.3.0', LOGS_2_3_0_PACKAGE_INFO); +packageInfoCache.set('docker-2.11.0', DOCKER_2_11_0_PACKAGE_INFO); describe('Fleet - templatePackagePolicyToFullInputStreams', () => { const mockInput: PackagePolicyInput = { @@ -330,6 +332,9 @@ describe('Fleet - getTemplateInputs', () => { if (packageInfo.name === 'log') { return LOGS_2_3_0_ASSETS_MAP; } + if (packageInfo.name === 'docker') { + return DOCKER_2_11_0_ASSETS_MAP; + } return new Map(); }); @@ -350,6 +355,14 @@ describe('Fleet - getTemplateInputs', () => { expect(template).toMatchSnapshot(); }); + it('should work for package with dynamic ids', async () => { + const soMock = savedObjectsClientMock.create(); + soMock.get.mockResolvedValue({ attributes: {} } as any); + const template = await getTemplateInputs(soMock, 'docker', '2.11.0', 'yml'); + + expect(template).toMatchSnapshot(); + }); + it('should work for input package', async () => { const soMock = savedObjectsClientMock.create(); soMock.get.mockResolvedValue({ attributes: {} } as any); From 6873bc6624e90fee280f8027633f448cc19922e9 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 6 Nov 2024 15:13:45 -0500 Subject: [PATCH 3/3] fix fixtures --- .../packages/__fixtures__/docker_2_11_0.ts | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/docker_2_11_0.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/docker_2_11_0.ts b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/docker_2_11_0.ts new file mode 100644 index 0000000000000..67bde1882cfed --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/docker_2_11_0.ts @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const DOCKER_2_11_0_PACKAGE_INFO = { + name: 'docker', + title: 'Docker', + version: '2.11.0', + release: 'ga', + description: 'Collect metrics and logs from Docker instances with Elastic Agent.', + type: 'integration', + download: '/epr/docker/docker-2.11.0.zip', + path: '/package/docker/2.11.0', + icons: [ + { + src: '/img/logo_docker.svg', + path: '/package/docker/2.11.0/img/logo_docker.svg', + title: 'logo docker', + size: '32x32', + type: 'image/svg+xml', + }, + ], + conditions: { + kibana: { + version: '^8.8.0', + }, + }, + owner: { + type: 'elastic', + github: 'elastic/obs-cloudnative-monitoring', + }, + categories: ['observability', 'containers'], + signature_path: '/epr/docker/docker-2.11.0.zip.sig', + format_version: '3.2.2', + readme: '/package/docker/2.11.0/docs/README.md', + license: 'basic', + screenshots: [ + { + src: '/img/docker-overview.png', + path: '/package/docker/2.11.0/img/docker-overview.png', + title: 'Docker Overview', + size: '5120x2562', + type: 'image/png', + }, + ], + assets: [ + '/package/docker/2.11.0/LICENSE.txt', + '/package/docker/2.11.0/changelog.yml', + '/package/docker/2.11.0/manifest.yml', + '/package/docker/2.11.0/docs/README.md', + '/package/docker/2.11.0/img/docker-overview.png', + '/package/docker/2.11.0/img/logo_docker.svg', + '/package/docker/2.11.0/data_stream/container/manifest.yml', + '/package/docker/2.11.0/data_stream/container/sample_event.json', + '/package/docker/2.11.0/data_stream/container_logs/manifest.yml', + '/package/docker/2.11.0/data_stream/container_logs/sample_event.json', + '/package/docker/2.11.0/data_stream/cpu/manifest.yml', + '/package/docker/2.11.0/data_stream/cpu/sample_event.json', + '/package/docker/2.11.0/data_stream/diskio/manifest.yml', + '/package/docker/2.11.0/data_stream/diskio/sample_event.json', + '/package/docker/2.11.0/data_stream/event/manifest.yml', + '/package/docker/2.11.0/data_stream/event/sample_event.json', + '/package/docker/2.11.0/data_stream/healthcheck/manifest.yml', + '/package/docker/2.11.0/data_stream/healthcheck/sample_event.json', + '/package/docker/2.11.0/data_stream/image/manifest.yml', + '/package/docker/2.11.0/data_stream/image/sample_event.json', + '/package/docker/2.11.0/data_stream/info/manifest.yml', + '/package/docker/2.11.0/data_stream/info/sample_event.json', + '/package/docker/2.11.0/data_stream/memory/manifest.yml', + '/package/docker/2.11.0/data_stream/memory/sample_event.json', + '/package/docker/2.11.0/data_stream/network/manifest.yml', + '/package/docker/2.11.0/data_stream/network/sample_event.json', + '/package/docker/2.11.0/kibana/dashboard/docker-AV4REOpp5NkDleZmzKkE.json', + '/package/docker/2.11.0/kibana/search/docker-Metrics-Docker.json', + '/package/docker/2.11.0/data_stream/container/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/container/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/container/fields/fields.yml', + '/package/docker/2.11.0/data_stream/container_logs/fields/agent.yml', + '/package/docker/2.11.0/data_stream/container_logs/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/container_logs/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/container_logs/fields/fields.yml', + '/package/docker/2.11.0/data_stream/cpu/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/cpu/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/cpu/fields/fields.yml', + '/package/docker/2.11.0/data_stream/diskio/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/diskio/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/diskio/fields/fields.yml', + '/package/docker/2.11.0/data_stream/event/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/event/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/event/fields/fields.yml', + '/package/docker/2.11.0/data_stream/healthcheck/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/healthcheck/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/healthcheck/fields/fields.yml', + '/package/docker/2.11.0/data_stream/image/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/image/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/image/fields/fields.yml', + '/package/docker/2.11.0/data_stream/info/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/info/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/info/fields/fields.yml', + '/package/docker/2.11.0/data_stream/memory/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/memory/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/memory/fields/fields.yml', + '/package/docker/2.11.0/data_stream/network/fields/base-fields.yml', + '/package/docker/2.11.0/data_stream/network/fields/ecs.yml', + '/package/docker/2.11.0/data_stream/network/fields/fields.yml', + '/package/docker/2.11.0/data_stream/container/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/container_logs/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/cpu/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/diskio/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/event/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/healthcheck/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/image/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/info/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/memory/agent/stream/stream.yml.hbs', + '/package/docker/2.11.0/data_stream/network/agent/stream/stream.yml.hbs', + ], + policy_templates: [ + { + name: 'docker', + title: 'Docker logs and metrics', + description: 'Collect logs and metrics from Docker instances', + inputs: [ + { + type: 'filestream', + title: 'Collect Docker container logs', + description: 'Collecting docker container logs', + }, + ], + multiple: true, + }, + ], + data_streams: [ + { + type: 'logs', + dataset: 'docker.container_logs', + title: 'Docker container logs', + release: 'ga', + streams: [ + { + input: 'filestream', + vars: [ + { + name: 'paths', + type: 'text', + title: 'Docker container log path', + multi: true, + required: true, + show_user: false, + default: ['/var/lib/docker/containers/${docker.container.id}/*-json.log'], + }, + { + name: 'containerParserStream', + type: 'text', + title: "Container parser's stream configuration", + multi: false, + required: true, + show_user: false, + default: 'all', + }, + { + name: 'condition', + type: 'text', + title: 'Condition', + description: + 'Condition to filter when to apply this datastream. Refer to [Docker provider](https://www.elastic.co/guide/en/fleet/current/docker-provider.html) to find the available keys and to [Conditions](https://www.elastic.co/guide/en/fleet/current/dynamic-input-configuration.html#conditions) on how to use the available keys in conditions.', + multi: false, + required: false, + show_user: true, + }, + { + name: 'additionalParsersConfig', + type: 'yaml', + title: 'Additional parsers configuration', + multi: false, + required: true, + show_user: false, + default: + "# - ndjson:\n# target: json\n# ignore_decoding_error: true\n# - multiline:\n# type: pattern\n# pattern: '^\\['\n# negate: true\n# match: after\n", + }, + { + name: 'processors', + type: 'yaml', + title: 'Processors', + description: + 'Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.', + multi: false, + required: false, + show_user: false, + }, + ], + template_path: 'stream.yml.hbs', + title: 'Collect Docker container logs', + description: 'Collect Docker container logs', + enabled: true, + }, + ], + package: 'docker', + elasticsearch: {}, + path: 'container_logs', + }, + ], +}; + +export const DOCKER_2_11_0_ASSETS_MAP = new Map([ + [ + 'docker-2.11.0/data_stream/container_logs/agent/stream/stream.yml.hbs', + Buffer.from(`id: docker-container-logs-\${docker.container.name}-\${docker.container.id} +paths: +{{#each paths}} + - {{this}} +{{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} +parsers: +- container: + stream: {{ containerParserStream }} + format: docker +{{ additionalParsersConfig }} + +{{#if processors}} +processors: +{{processors}} +{{/if}} +`), + ], +]);