From 0855613002e5806817395cf593d1f10d4cc495d9 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 13 Oct 2023 13:07:22 +0200 Subject: [PATCH 1/8] [Fleet] Add support for additional types for dynamic mappings Add support for fields that generate and dynamic mapping of the following types: - scaled_float - half_float - match_only_text - aggregate_metric_double - ip - integer (mapped as long as in other cases) - group (child fields are installed as dynamic mappings) When the type is not known, an error is raised now instead of silently ignoring the field definition. --- .../elasticsearch/template/template.test.ts | 99 +++++++++++++++++++ .../epm/elasticsearch/template/template.ts | 59 ++++++++--- 2 files changed, 146 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index e5c323de62dfd..cfa108c559e00 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -1124,6 +1124,105 @@ describe('EPM template', () => { expect(mappings).toEqual(runtimeFieldMapping); }); + it('tests processing scaled_float fields in a dynamic template', () => { + const textWithRuntimeFieldsLiteralYml = ` +- name: numeric_labels + type: object + object_type: scaled_float +`; + const runtimeFieldMapping = { + properties: {}, + dynamic_templates: [ + { + 'numeric_labels': { + match_mapping_type: '*', + path_match: 'numeric_labels.*', + mapping: { + type: 'scaled_float', + scaling_factor: 1000, + }, + }, + }, + ], + }; + const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(runtimeFieldMapping); + }); + + it('tests processing aggregate_metric_double fields in a dynamic template', () => { + const textWithRuntimeFieldsLiteralYml = ` +- name: aggregate.* + type: aggregate_metric_double + metrics: ["min", "max", "sum", "value_count"] + default_metric: "max" +`; + const runtimeFieldMapping = { + properties: {}, + dynamic_templates: [ + { + 'aggregate.*': { + match_mapping_type: '*', + path_match: 'aggregate.*', + mapping: { + type: 'aggregate_metric_double', + metrics: ["min", "max", "sum", "value_count"], + default_metric: "max" + }, + }, + }, + ], + }; + const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(runtimeFieldMapping); + }); + + it('tests processing groub sub fields in a dynamic template', () => { + const textWithRuntimeFieldsLiteralYml = ` +- name: group.*.network + type: group + fields: + - name: bytes + type: integer + metric_type: counter +`; + const runtimeFieldMapping = { + properties: {}, + dynamic_templates: [ + { + 'group.*.network.bytes': { + match_mapping_type: 'long', + path_match: 'group.*.network.bytes', + mapping: { + type: 'long', + time_series_metric: 'counter', + }, + }, + }, + ], + }; + const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(runtimeFieldMapping); + }); + + it('tests unexpected type for field as dynamic template fails', () => { + const textWithRuntimeFieldsLiteralYml = ` +- name: labels.* + type: object + object_type: constant_keyword +`; + const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml); + expect(() => { + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + }).toThrow(); + }); + it('tests priority and index pattern for data stream without dataset_is_prefix', () => { const dataStreamDatasetIsPrefixUnset = { type: 'metrics', diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index e55cad3f80a9c..2c8a8f1578dfc 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -216,10 +216,8 @@ function _generateMappings( case 'double': case 'long': case 'boolean': - dynProperties = { - type: field.object_type, - time_series_metric: field.metric_type, - }; + dynProperties.type = field.object_type; + dynProperties.time_series_metric = field.metric_type; matchingType = field.object_type_mapping_type ?? field.object_type; default: break; @@ -258,27 +256,64 @@ function _generateMappings( dynProperties = histogram(field); matchingType = field.object_type_mapping_type ?? '*'; break; + case 'ip': + case 'keyword': + case 'match_only_text': case 'text': + case 'wildcard': dynProperties.type = field.object_type; matchingType = field.object_type_mapping_type ?? 'string'; break; - case 'keyword': + case 'scaled_float': + dynProperties = scaledFloat(field); + matchingType = field.object_type_mapping_type ?? '*'; + break; + case 'aggregate_metric_double': dynProperties.type = field.object_type; - matchingType = field.object_type_mapping_type ?? 'string'; + dynProperties.metrics = field.metrics; + dynProperties.default_metric = field.default_metric; + matchingType = field.object_type_mapping_type ?? '*'; break; - case 'byte': case 'double': case 'float': + case 'half_float': + dynProperties.type = field.object_type; + dynProperties.time_series_metric = field.metric_type; + matchingType = field.object_type_mapping_type ?? 'double'; + break; + case 'byte': case 'long': case 'short': + dynProperties.type = field.object_type; + dynProperties.time_series_metric = field.metric_type; + matchingType = field.object_type_mapping_type ?? 'long'; + break; + case 'integer': + // Map integers as long, as in other cases. + dynProperties.type = 'long'; + dynProperties.time_series_metric = field.metric_type; + matchingType = field.object_type_mapping_type ?? 'long'; + break; case 'boolean': - dynProperties = { - type: field.object_type, - time_series_metric: field.metric_type, - }; + dynProperties.type = field.object_type; + dynProperties.time_series_metric = field.metric_type; matchingType = field.object_type_mapping_type ?? field.object_type; - default: break; + case 'group': + const fields = field.fields.map((field) => ({ + ...field, + type: 'object', + object_type: field.object_type ?? field.type, + })) + _generateMappings(fields, { + ...ctx, + groupFieldName: ctx.groupFieldName + ? `${ctx.groupFieldName}.${field.name}` + : field.name, + }); + break; + default: + throw new Error(`no dynamic mapping generated for field ${path} of type ${field.object_type}`); } if (dynProperties && matchingType) { From 0827919e9d2db73a56f097d66a05dc7471ff8584 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 13 Oct 2023 15:35:44 +0200 Subject: [PATCH 2/8] Fix linting --- .../server/services/epm/elasticsearch/template/template.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 2c8a8f1578dfc..beaf0d286ed65 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -300,10 +300,10 @@ function _generateMappings( matchingType = field.object_type_mapping_type ?? field.object_type; break; case 'group': - const fields = field.fields.map((field) => ({ - ...field, + const fields = field.fields.map((dynField) => ({ + ...dynField, type: 'object', - object_type: field.object_type ?? field.type, + object_type: dynField.object_type ?? dynField.type, })) _generateMappings(fields, { ...ctx, From 2b454ab16cd840f0146548a3837931fd1baf2837 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 13 Oct 2023 14:16:14 +0000 Subject: [PATCH 3/8] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../elasticsearch/template/template.test.ts | 6 +++--- .../epm/elasticsearch/template/template.ts | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index cfa108c559e00..72e4262b20741 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -1134,7 +1134,7 @@ describe('EPM template', () => { properties: {}, dynamic_templates: [ { - 'numeric_labels': { + numeric_labels: { match_mapping_type: '*', path_match: 'numeric_labels.*', mapping: { @@ -1167,8 +1167,8 @@ describe('EPM template', () => { path_match: 'aggregate.*', mapping: { type: 'aggregate_metric_double', - metrics: ["min", "max", "sum", "value_count"], - default_metric: "max" + metrics: ['min', 'max', 'sum', 'value_count'], + default_metric: 'max', }, }, }, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index beaf0d286ed65..2d97f432158af 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -206,7 +206,7 @@ function _generateMappings( if (type === 'object' && field.object_type) { const pathMatch = path.includes('*') ? path : `${path}.*`; - let dynProperties: Properties = getDefaultProperties(field); + const dynProperties: Properties = getDefaultProperties(field); let matchingType: string | undefined; switch (field.object_type) { case 'keyword': @@ -300,12 +300,12 @@ function _generateMappings( matchingType = field.object_type_mapping_type ?? field.object_type; break; case 'group': - const fields = field.fields.map((dynField) => ({ - ...dynField, - type: 'object', - object_type: dynField.object_type ?? dynField.type, - })) - _generateMappings(fields, { + const fields = field.fields.map((dynField) => ({ + ...dynField, + type: 'object', + object_type: dynField.object_type ?? dynField.type, + })); + _generateMappings(fields, { ...ctx, groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` @@ -313,7 +313,9 @@ function _generateMappings( }); break; default: - throw new Error(`no dynamic mapping generated for field ${path} of type ${field.object_type}`); + throw new Error( + `no dynamic mapping generated for field ${path} of type ${field.object_type}` + ); } if (dynProperties && matchingType) { From 5beeadf175608585b06ff452c89a84774036196a Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 13 Oct 2023 17:45:32 +0200 Subject: [PATCH 4/8] Linting and support for flattened --- .../epm/elasticsearch/template/template.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index beaf0d286ed65..6c032daf8c687 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -300,18 +300,22 @@ function _generateMappings( matchingType = field.object_type_mapping_type ?? field.object_type; break; case 'group': - const fields = field.fields.map((dynField) => ({ - ...dynField, - type: 'object', - object_type: dynField.object_type ?? dynField.type, - })) - _generateMappings(fields, { + const subFields = field.fields.map((subField) => ({ + ...subField, + type: 'object', + object_type: subField.object_type ?? subField.type, + })) + _generateMappings(subFields, { ...ctx, groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name, }); break; + case 'flattened': + dynProperties.type = field.object_type; + matchingType = field.object_type_mapping_type ?? 'object'; + break; default: throw new Error(`no dynamic mapping generated for field ${path} of type ${field.object_type}`); } From 77c4c013cbff2184edf757cc24c7c861cafd19bd Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 13 Oct 2023 15:54:29 +0000 Subject: [PATCH 5/8] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../server/services/epm/elasticsearch/template/template.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index f23268f5777c4..ecc9e033111f9 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -304,7 +304,7 @@ function _generateMappings( ...subField, type: 'object', object_type: subField.object_type ?? subField.type, - })) + })); _generateMappings(subFields, { ...ctx, groupFieldName: ctx.groupFieldName From 7a54b240345260c5d5b1e28149c836783dc3849e Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 13 Oct 2023 18:27:10 +0200 Subject: [PATCH 6/8] More linting --- .../services/epm/elasticsearch/template/template.test.ts | 2 +- .../server/services/epm/elasticsearch/template/template.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 72e4262b20741..f0459b10c8570 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -1219,7 +1219,7 @@ describe('EPM template', () => { const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml); expect(() => { const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); + generateMappings(processedFields); }).toThrow(); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index ecc9e033111f9..a13d0155f44ca 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -300,6 +300,9 @@ function _generateMappings( matchingType = field.object_type_mapping_type ?? field.object_type; break; case 'group': + if (!field?.fields) { + break; + } const subFields = field.fields.map((subField) => ({ ...subField, type: 'object', From 445018cbcaab71a1739db05c691fd4a3fd56cb45 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 16 Oct 2023 11:11:51 +0200 Subject: [PATCH 7/8] Add unsigned long --- .../fleet/server/services/epm/elasticsearch/template/template.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index a13d0155f44ca..c9b99d77b5b2a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -284,6 +284,7 @@ function _generateMappings( case 'byte': case 'long': case 'short': + case 'unsigned_long': dynProperties.type = field.object_type; dynProperties.time_series_metric = field.metric_type; matchingType = field.object_type_mapping_type ?? 'long'; From 2d1dfd4dd66bf759f8f75a7aaebd608f9199b84f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 16 Oct 2023 11:12:15 +0200 Subject: [PATCH 8/8] Linting --- .../server/services/epm/elasticsearch/template/template.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index c9b99d77b5b2a..36e13ae331f58 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -284,7 +284,7 @@ function _generateMappings( case 'byte': case 'long': case 'short': - case 'unsigned_long': + case 'unsigned_long': dynProperties.type = field.object_type; dynProperties.time_series_metric = field.metric_type; matchingType = field.object_type_mapping_type ?? 'long';