diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index da068809234f4..ea615318fe696 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -58,7 +58,22 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = ` exports[`EPM template tests loading cockroachdb_dynamic_templates.yml: cockroachdb_dynamic_templates.yml 1`] = ` { - "properties": {}, + "properties": { + "cockroachdb": { + "properties": { + "status": { + "properties": { + "labels": { + "type": "object", + "dynamic": true + } + }, + "type": "object", + "dynamic": true + } + } + } + }, "dynamic_templates": [ { "cockroachdb.status.labels": { @@ -78,6 +93,16 @@ exports[`EPM template tests loading cockroachdb_dynamic_templates.yml: cockroach "path_match": "cockroachdb.status.*.value" } }, + { + "cockroachdb.status.*": { + "mapping": { + "type": "object", + "dynamic": true + }, + "match_mapping_type": "object", + "path_match": "cockroachdb.status.*" + } + }, { "cockroachdb.status.*.counter": { "mapping": { @@ -837,39 +862,24 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "network_summary": { "properties": { "ip": { - "properties": { - "*": { - "type": "object" - } - } + "type": "object", + "dynamic": true }, "tcp": { - "properties": { - "*": { - "type": "object" - } - } + "type": "object", + "dynamic": true }, "udp": { - "properties": { - "*": { - "type": "object" - } - } + "type": "object", + "dynamic": true }, "udp_lite": { - "properties": { - "*": { - "type": "object" - } - } + "type": "object", + "dynamic": true }, "icmp": { - "properties": { - "*": { - "type": "object" - } - } + "type": "object", + "dynamic": true } } }, @@ -883,6 +893,10 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "type": "keyword", "ignore_above": 2048 }, + "env": { + "type": "object", + "dynamic": true + }, "cpu": { "properties": { "user": { @@ -1076,8 +1090,14 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } } + }, + "percpu": { + "type": "object", + "dynamic": true } - } + }, + "type": "object", + "dynamic": true }, "memory": { "properties": { @@ -1355,7 +1375,9 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } } - } + }, + "type": "object", + "dynamic": true }, "raid": { "properties": { @@ -1388,8 +1410,14 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` }, "failed": { "type": "long" + }, + "states": { + "type": "object", + "dynamic": true } - } + }, + "type": "object", + "dynamic": true }, "blocks": { "properties": { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 59e5c68fb7345..96a2547c9e58d 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -1151,7 +1151,12 @@ describe('EPM template', () => { runtime: true `; const runtimeFieldMapping = { - properties: {}, + properties: { + labels: { + type: 'object', + dynamic: true, + }, + }, dynamic_templates: [ { 'labels.*': { @@ -1177,7 +1182,12 @@ describe('EPM template', () => { object_type: scaled_float `; const runtimeFieldMapping = { - properties: {}, + properties: { + numeric_labels: { + type: 'object', + dynamic: true, + }, + }, dynamic_templates: [ { numeric_labels: { @@ -1205,7 +1215,12 @@ describe('EPM template', () => { default_metric: "max" `; const runtimeFieldMapping = { - properties: {}, + properties: { + aggregate: { + type: 'object', + dynamic: true, + }, + }, dynamic_templates: [ { 'aggregate.*': { @@ -1226,7 +1241,7 @@ describe('EPM template', () => { expect(mappings).toEqual(runtimeFieldMapping); }); - it('tests processing groub sub fields in a dynamic template', () => { + it('tests processing group sub fields in a dynamic template', () => { const textWithRuntimeFieldsLiteralYml = ` - name: group.*.network type: group @@ -1236,7 +1251,12 @@ describe('EPM template', () => { metric_type: counter `; const runtimeFieldMapping = { - properties: {}, + properties: { + group: { + type: 'object', + dynamic: true, + }, + }, dynamic_templates: [ { 'group.*.network.bytes': { @@ -1248,6 +1268,26 @@ describe('EPM template', () => { }, }, }, + { + 'group.*.network': { + path_match: 'group.*.network', + match_mapping_type: 'object', + mapping: { + type: 'object', + dynamic: true, + }, + }, + }, + { + 'group.*': { + path_match: 'group.*', + match_mapping_type: 'object', + mapping: { + type: 'object', + dynamic: true, + }, + }, + }, ], }; const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 26c6926c0bbc4..622538e152836 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -151,8 +151,8 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings { path: string; matchingType: string; pathMatch: string; - properties: string; - runtimeProperties?: string; + properties: Properties; + runtimeProperties?: Properties; }) => { const name = dynamicMapping.path; if (dynamicTemplateNames.has(name)) { @@ -210,9 +210,64 @@ function _generateMappings( ): { properties: IndexTemplateMappings['properties']; hasNonDynamicTemplateMappings: boolean; + hasDynamicTemplateMappings: boolean; } { let hasNonDynamicTemplateMappings = false; + let hasDynamicTemplateMappings = false; const props: Properties = {}; + + function addParentObjectAsStaticProperty(field: Field) { + // Don't add intermediate objects for wildcard names, as it will + // be added for its parent object. + if (field.name.includes('*')) { + return; + } + + const fieldProps = { + type: 'object', + dynamic: true, + }; + + props[field.name] = fieldProps; + hasNonDynamicTemplateMappings = true; + } + + function addDynamicMappingWithIntermediateObjects( + path: string, + pathMatch: string, + matchingType: string, + dynProperties: Properties, + fieldProps?: Properties + ) { + ctx.addDynamicMapping({ + path, + pathMatch, + matchingType, + properties: dynProperties, + runtimeProperties: fieldProps, + }); + hasDynamicTemplateMappings = true; + + // Add dynamic intermediate objects. + const parts = pathMatch.split('.'); + for (let i = parts.length - 1; i > 0; i--) { + const name = parts.slice(0, i).join('.'); + if (!name.includes('*')) { + continue; + } + const dynProps: Properties = { + type: 'object', + dynamic: true, + }; + ctx.addDynamicMapping({ + path: name, + pathMatch: name, + matchingType: 'object', + properties: dynProps, + }); + } + } + // TODO: this can happen when the fields property in fields.yml is present but empty // Maybe validation should be moved to fields/field.ts if (fields) { @@ -250,13 +305,17 @@ function _generateMappings( const fieldProps = generateRuntimeFieldProps(_field); if (dynProperties && matchingType) { - ctx.addDynamicMapping({ + addDynamicMappingWithIntermediateObjects( path, pathMatch, matchingType, - properties: dynProperties, - runtimeProperties: fieldProps, - }); + dynProperties, + fieldProps + ); + + // Add the parent object as static property, this is needed for + // index templates not using `"dynamic": true`. + addParentObjectAsStaticProperty(field); } return; } @@ -331,12 +390,15 @@ function _generateMappings( type: 'object', object_type: subField.object_type ?? subField.type, })); - _generateMappings(subFields, { + const mappings = _generateMappings(subFields, { ...ctx, groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name, }); + if (mappings.hasDynamicTemplateMappings) { + hasDynamicTemplateMappings = true; + } break; case 'flattened': dynProperties.type = field.object_type; @@ -349,12 +411,11 @@ function _generateMappings( } if (dynProperties && matchingType) { - ctx.addDynamicMapping({ - path, - pathMatch, - matchingType, - properties: dynProperties, - }); + addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties); + + // Add the parent object as static property, this is needed for + // index templates not using `"dynamic": true`. + addParentObjectAsStaticProperty(field); } } else { let fieldProps = getDefaultProperties(field); @@ -367,14 +428,25 @@ function _generateMappings( ? `${ctx.groupFieldName}.${field.name}` : field.name, }); - if (!mappings.hasNonDynamicTemplateMappings) { + if (mappings.hasNonDynamicTemplateMappings) { + fieldProps = { + properties: + Object.keys(mappings.properties).length > 0 ? mappings.properties : undefined, + ...generateDynamicAndEnabled(field), + }; + if (mappings.hasDynamicTemplateMappings) { + fieldProps.type = 'object'; + fieldProps.dynamic = true; + } + } else if (mappings.hasDynamicTemplateMappings) { + fieldProps = { + type: 'object', + dynamic: true, + }; + hasDynamicTemplateMappings = true; + } else { return; } - - fieldProps = { - properties: mappings.properties, - ...generateDynamicAndEnabled(field), - }; break; case 'group-nested': fieldProps = { @@ -478,13 +550,25 @@ function _generateMappings( fieldProps.time_series_dimension = field.dimension; } - props[field.name] = fieldProps; + // Even if we don't add the property because it has a wildcard, notify + // the parent that there is some kind of property, so the intermediate object + // is still created. + // This is done for legacy packages that include ambiguous mappings with objects + // without object type. This is not allowed starting on Package Spec v3. hasNonDynamicTemplateMappings = true; + + // Avoid including maps with wildcards, they have generated dynamic mappings. + if (field.name.includes('*')) { + hasDynamicTemplateMappings = true; + return; + } + + props[field.name] = fieldProps; } }); } - return { properties: props, hasNonDynamicTemplateMappings }; + return { properties: props, hasNonDynamicTemplateMappings, hasDynamicTemplateMappings }; } function generateDynamicAndEnabled(field: Field) {