diff --git a/docs/changelog/112444.yaml b/docs/changelog/112444.yaml new file mode 100644 index 0000000000000..bfa4fd693f0e0 --- /dev/null +++ b/docs/changelog/112444.yaml @@ -0,0 +1,6 @@ +pr: 112444 +summary: Full coverage of ECS by ecs@mappings when `date_detection` is disabled +area: Mapping +type: bug +issues: + - 112398 diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json index 138a8d763bae6..997bd942881ad 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json @@ -155,7 +155,11 @@ "ingested", "*.ingested", "*.start", - "*.end" + "*.end", + "*.indicator.first_seen", + "*.indicator.last_seen", + "*.indicator.modified_at", + "*threat.enrichments.matched.occurred" ], "unmatch_mapping_type": "object" } diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java index 423865eaab617..eba239cd97e28 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.cluster.ElasticsearchCluster; @@ -206,6 +207,44 @@ public void testNumericMessage() throws IOException { verifyEcsMappings(indexName); } + public void testDateFieldsWithDifferentFormats() throws IOException { + Map dateFieldsMap = ecsFlatFieldDefinitions.entrySet() + .stream() + .filter(entry -> "date".equals(entry.getValue().get("type"))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // test with iso8601 format + String indexName = "test-date-fields-as-is8601"; + createTestIndex(indexName); + Map document = new HashMap<>(); + DateFormatter formatter = DateFormatter.forPattern(FormatNames.ISO8601.getName()); + for (String field : dateFieldsMap.keySet()) { + document.put(field, formatter.formatMillis(System.currentTimeMillis())); + } + verifyAllDateFields(indexName, document, dateFieldsMap); + + // test with milliseconds since epoch format + indexName = "test-date-fields-as-millis"; + createTestIndex(indexName); + document = new HashMap<>(); + for (String field : dateFieldsMap.keySet()) { + document.put(field, System.currentTimeMillis()); + } + verifyAllDateFields(indexName, document, dateFieldsMap); + } + + private void verifyAllDateFields(String indexName, Map document, Map dateFieldsMap) throws IOException { + indexDocument(indexName, document); + final Map rawMappings = getMappings(indexName); + final Map> flatFieldMappings = new HashMap<>(); + processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), ""); + flatFieldMappings.forEach((fieldName, fieldMappings) -> { + if (dateFieldsMap.containsKey(fieldName)) { + assertType("date", fieldMappings); + } + }); + } + private void assertType(String expectedType, Map actualMappings) { assertNotNull("expected to get non-null mappings for field", actualMappings); assertEquals(expectedType, actualMappings.get("type")); @@ -312,6 +351,7 @@ private static void createTestIndex(String indexName, @Nullable Map { - return DateFormatter.forPattern("strict_date_optional_time").formatMillis(System.currentTimeMillis()); + return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).formatMillis(System.currentTimeMillis()); } case "ip" -> { return NetworkAddress.format(randomIp(true)); @@ -485,9 +525,11 @@ private void verifyEcsMappings(String indexName, String fieldPrefix) throws IOEx ); }); fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> { - Map fieldMappings = ecsFlatFieldDefinitions.get(fieldName); + // if fieldPrefix is not null, we need to remove it from the field name for the ECS lookup + String ecsFieldName = fieldPrefix == null ? fieldName : fieldName.substring(fieldPrefix.length()); + Map fieldMappings = ecsFlatFieldDefinitions.get(ecsFieldName); if (fieldMappings == null) { - fieldMappings = ecsFlatMultiFieldDefinitions.get(fieldName); + fieldMappings = ecsFlatMultiFieldDefinitions.get(ecsFieldName); } String ecsExpectedType = (String) fieldMappings.get("type"); logger.error( diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 9e847455d2c86..592842f61eee8 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -48,7 +48,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { // The stack template registry version. This number must be incremented when we make changes // to built-in templates. - public static final int REGISTRY_VERSION = 13; + public static final int REGISTRY_VERSION = 14; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting(