diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java index f22349008e0f..435e141abc46 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java @@ -61,7 +61,8 @@ public class RollupField { NUMERIC_FIELD_MAPPER_TYPES = types; } - public static final String DATE_FIELD_MAPPER_TYPE = DateFieldMapper.CONTENT_TYPE; + public static final List DATE_FIELD_MAPPER_TYPES = List.of(DateFieldMapper.CONTENT_TYPE, + DateFieldMapper.DATE_NANOS_CONTENT_TYPE); /** * Format to the appropriate Rollup field name convention diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java index e8f67c8ac393..b146e3b0fde8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.xpack.core.rollup.RollupField; import java.io.IOException; import java.time.ZoneId; @@ -314,24 +315,31 @@ public String getIntervalTypeName() { public void validateMappings(Map> fieldCapsResponse, ActionRequestValidationException validationException) { - Map fieldCaps = fieldCapsResponse.get(field); if (fieldCaps != null && fieldCaps.isEmpty() == false) { - if (fieldCaps.containsKey("date") && fieldCaps.size() == 1) { - if (fieldCaps.get("date").isAggregatable()) { - return; - } else { - validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " + - "but is not."); + boolean matchesDateType = false; + for (String dateType : RollupField.DATE_FIELD_MAPPER_TYPES) { + if (fieldCaps.containsKey(dateType) && fieldCaps.size() == 1) { + matchesDateType |= true; + if (fieldCaps.get(dateType).isAggregatable()) { + return; + } else { + validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " + + "but is not."); + } } - - } else { - validationException.addValidationError("The field referenced by a date_histo group must be a [date] type across all " + - "indices in the index pattern. Found: " + fieldCaps.keySet().toString() + " for field [" + field + "]"); } - } - validationException.addValidationError("Could not find a [date] field with name [" + field + "] in any of the indices matching " + + if (matchesDateType == false) { + validationException.addValidationError("The field referenced by a date_histo group must be one of type [" + + Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] across all " + + "indices in the index pattern. Found: " + fieldCaps.keySet().toString() + " for field [" + field + "]"); + } + } else { + validationException.addValidationError("Could not find one of [" + + Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] fields with name [" + + field + "] in any of the indices matching " + "the index pattern."); + } } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java index 47b45dc69dd2..f0b8bc4cf9ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java @@ -115,19 +115,21 @@ public void validateMappings(Map> fieldCa } if (RollupField.NUMERIC_FIELD_MAPPER_TYPES.contains(key)) { // nothing to do as all metrics are supported by SUPPORTED_NUMERIC_METRICS currently - } else if (RollupField.DATE_FIELD_MAPPER_TYPE.equals(key)) { + } else if (RollupField.DATE_FIELD_MAPPER_TYPES.contains(key)) { if (RollupField.SUPPORTED_DATE_METRICS.containsAll(metrics) == false) { validationException.addValidationError( - buildSupportedMetricError("date", RollupField.SUPPORTED_DATE_METRICS)); + buildSupportedMetricError(key, RollupField.SUPPORTED_DATE_METRICS)); } } else { - validationException.addValidationError("The field referenced by a metric group must be a [numeric] or [date] type, " + + validationException.addValidationError("The field referenced by a metric group must be a [numeric] or [" + + Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] type, " + "but found " + fieldCaps.keySet().toString() + " for field [" + field + "]"); } }); } else { - validationException.addValidationError("Could not find a [numeric] or [date] field with name [" + field + "] in any of the " + - "indices matching the index pattern."); + validationException.addValidationError("Could not find a [numeric] or [" + + Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + + "] field with name [" + field + "] in any of the " + "indices matching the index pattern."); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java index 3535cb1ed55a..eef24b48d2db 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Random; @@ -29,6 +30,7 @@ import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween; import static com.carrotsearch.randomizedtesting.generators.RandomPicks.randomFrom; import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween; +import static org.elasticsearch.test.ESTestCase.randomSubsetOf; import static org.elasticsearch.test.ESTestCase.randomZone; public class ConfigTestHelpers { @@ -69,7 +71,10 @@ public static GroupConfig randomGroupConfig(final Random random) { } public static DateHistogramGroupConfig randomDateHistogramGroupConfig(final Random random) { - final String field = randomField(random); + return randomDateHistogramGroupConfigWithField(random, randomField(random)); + } + + public static DateHistogramGroupConfig randomDateHistogramGroupConfigWithField(final Random random, final String field) { final DateHistogramInterval delay = random.nextBoolean() ? randomInterval() : null; final String timezone = random.nextBoolean() ? randomZone().getId() : null; if (random.nextBoolean()) { @@ -128,26 +133,11 @@ public static List randomMetricsConfigs(final Random random) { public static MetricConfig randomMetricConfig(final Random random) { final String field = randomAsciiAlphanumOfLengthBetween(random, 15, 25); // large names so we don't accidentally collide - final List metrics = new ArrayList<>(); - if (random.nextBoolean()) { - metrics.add("min"); - } - if (random.nextBoolean()) { - metrics.add("max"); - } - if (random.nextBoolean()) { - metrics.add("sum"); - } - if (random.nextBoolean()) { - metrics.add("avg"); - } - if (random.nextBoolean()) { - metrics.add("value_count"); - } - if (metrics.size() == 0) { - metrics.add("min"); - } - return new MetricConfig(field, Collections.unmodifiableList(metrics)); + return randomMetricConfigWithFieldAndMetrics(random, field, RollupField.SUPPORTED_METRICS); + } + + public static MetricConfig randomMetricConfigWithFieldAndMetrics(final Random random, String field, Collection metrics) { + return new MetricConfig(field, Collections.unmodifiableList(randomSubsetOf(randomIntBetween(random, 1, metrics.size()), metrics))); } public static TermsGroupConfig randomTermsGroupConfig(final Random random) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java index 65844e9e1ca9..75936b0d193b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java @@ -68,8 +68,8 @@ public void testValidateNoMapping() { DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field", new DateHistogramInterval("1d"), null, null); config.validateMappings(responseMap, e); - assertThat(e.validationErrors().get(0), equalTo("Could not find a [date] field with name [my_field] in any of the " + - "indices matching the index pattern.")); + assertThat(e.validationErrors().get(0), equalTo("Could not find one of [date,date_nanos] fields with name [my_field] in " + + "any of the indices matching the index pattern.")); } public void testValidateNomatchingField() { @@ -83,8 +83,8 @@ public void testValidateNomatchingField() { DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field", new DateHistogramInterval("1d"), null, null); config.validateMappings(responseMap, e); - assertThat(e.validationErrors().get(0), equalTo("Could not find a [date] field with name [my_field] in any of the " + - "indices matching the index pattern.")); + assertThat(e.validationErrors().get(0), equalTo("Could not find one of [date,date_nanos] fields with name [my_field] in " + + "any of the indices matching the index pattern.")); } public void testValidateFieldWrongType() { @@ -98,8 +98,8 @@ public void testValidateFieldWrongType() { DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field", new DateHistogramInterval("1d"), null, null); config.validateMappings(responseMap, e); - assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be a [date] type across all " + - "indices in the index pattern. Found: [keyword] for field [my_field]")); + assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be one of type " + + "[date,date_nanos] across all indices in the index pattern. Found: [keyword] for field [my_field]")); } public void testValidateFieldMixtureTypes() { @@ -116,8 +116,8 @@ public void testValidateFieldMixtureTypes() { DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field", new DateHistogramInterval("1d"), null, null); config.validateMappings(responseMap, e); - assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be a [date] type across all " + - "indices in the index pattern. Found: [date, keyword] for field [my_field]")); + assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be one of type " + + "[date,date_nanos] across all indices in the index pattern. Found: [date, keyword] for field [my_field]")); } public void testValidateFieldMatchingNotAggregatable() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java index 92bbd0d723de..8bb5c178a177 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java @@ -14,10 +14,11 @@ import org.elasticsearch.xpack.core.rollup.RollupField; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.equalTo; @@ -49,11 +50,11 @@ public void testValidateNoMapping() { MetricConfig config = new MetricConfig("my_field", singletonList("max")); config.validateMappings(responseMap, e); - assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date] field with name [my_field] in any" + - " of the indices matching the index pattern.")); + assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date,date_nanos] field with name [my_field] " + + "in any of the indices matching the index pattern.")); } - public void testValidateNomatchingField() { + public void testValidateNoMatchingField() { ActionRequestValidationException e = new ActionRequestValidationException(); Map> responseMap = new HashMap<>(); @@ -63,8 +64,8 @@ public void testValidateNomatchingField() { MetricConfig config = new MetricConfig("my_field", singletonList("max")); config.validateMappings(responseMap, e); - assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date] field with name [my_field] in any" + - " of the indices matching the index pattern.")); + assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date,date_nanos] field with name [my_field] " + + "in any of the indices matching the index pattern.")); } public void testValidateFieldWrongType() { @@ -77,7 +78,7 @@ public void testValidateFieldWrongType() { MetricConfig config = new MetricConfig("my_field", singletonList("max")); config.validateMappings(responseMap, e); - assertThat("The field referenced by a metric group must be a [numeric] or [date] type," + + assertThat("The field referenced by a metric group must be a [numeric] or [date,date_nanos] type," + " but found [keyword] for field [my_field]", is(in(e.validationErrors()))); } @@ -95,90 +96,54 @@ public void testValidateFieldMatchingNotAggregatable() { assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not.")); } - public void testValidateDateFieldUnsupportedMetric() { - ActionRequestValidationException e = new ActionRequestValidationException(); + public void testValidateDateFieldsUnsupportedMetric() { Map> responseMap = new HashMap<>(); - // Have to mock fieldcaps because the ctor's aren't public... - FieldCapabilities fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("date", fieldCaps)); + for (String mappingType : RollupField.DATE_FIELD_MAPPER_TYPES) { + // Have to mock fieldcaps because the ctor's aren't public... + FieldCapabilities fieldCaps = mock(FieldCapabilities.class); + when(fieldCaps.isAggregatable()).thenReturn(true); + responseMap.put("my_field", Collections.singletonMap(mappingType, fieldCaps)); + + Set unsupportedMetrics = new HashSet<>(RollupField.SUPPORTED_METRICS); + unsupportedMetrics.removeAll(RollupField.SUPPORTED_DATE_METRICS); + for (String unsupportedMetric : unsupportedMetrics) { + MetricConfig config = new MetricConfig("my_field", Collections.singletonList(unsupportedMetric)); + ActionRequestValidationException e = new ActionRequestValidationException(); + config.validateMappings(responseMap, e); + assertThat(e.validationErrors().get(0), equalTo("Only the metrics " + RollupField.SUPPORTED_DATE_METRICS.toString() + + " are supported for [" + mappingType + "] types, but unsupported metrics [" + unsupportedMetric + + "] supplied for field [my_field]")); + } + } - MetricConfig config = new MetricConfig("my_field", Arrays.asList("avg", "max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().get(0), equalTo("Only the metrics " + RollupField.SUPPORTED_DATE_METRICS.toString() + - " are supported for [date] types, but unsupported metrics [avg] supplied for field [my_field]")); } public void testValidateMatchingField() { ActionRequestValidationException e = new ActionRequestValidationException(); Map> responseMap = new HashMap<>(); - // Have to mock fieldcaps because the ctor's aren't public... - FieldCapabilities fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("long", fieldCaps)); - - MetricConfig config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); - - - fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("double", fieldCaps)); - config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); - - fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("float", fieldCaps)); - config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); - - fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("short", fieldCaps)); - config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); - - fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("byte", fieldCaps)); - config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); - - fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("half_float", fieldCaps)); - config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); - - fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("scaled_float", fieldCaps)); - config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); - - fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("integer", fieldCaps)); - config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); - - fieldCaps = mock(FieldCapabilities.class); - when(fieldCaps.isAggregatable()).thenReturn(true); - responseMap.put("my_field", Collections.singletonMap("date", fieldCaps)); - config = new MetricConfig("my_field", singletonList("max")); - config.validateMappings(responseMap, e); - assertThat(e.validationErrors().size(), equalTo(0)); + for (String numericType : RollupField.NUMERIC_FIELD_MAPPER_TYPES) { + // Have to mock fieldcaps because the ctor's aren't public... + FieldCapabilities fieldCaps = mock(FieldCapabilities.class); + when(fieldCaps.isAggregatable()).thenReturn(true); + responseMap.put("my_field", Collections.singletonMap(numericType, fieldCaps)); + MetricConfig config = ConfigTestHelpers + .randomMetricConfigWithFieldAndMetrics(random(), "my_field", RollupField.SUPPORTED_NUMERIC_METRICS); + config.validateMappings(responseMap, e); + assertThat(e.validationErrors().size(), equalTo(0)); + } + + for (String dateType : RollupField.DATE_FIELD_MAPPER_TYPES) { + // Have to mock fieldcaps because the ctor's aren't public... + FieldCapabilities fieldCaps = mock(FieldCapabilities.class); + when(fieldCaps.isAggregatable()).thenReturn(true); + responseMap.put("my_field", Collections.singletonMap(dateType, fieldCaps)); + MetricConfig config = ConfigTestHelpers + .randomMetricConfigWithFieldAndMetrics(random(), "my_field", RollupField.SUPPORTED_DATE_METRICS); + config.validateMappings(responseMap, e); + assertThat(e.validationErrors().size(), equalTo(0)); + } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml index ee076e213b15..4542e1b24a2c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml @@ -208,7 +208,7 @@ setup: "Validation failures": - do: - catch: /Could not find a \[numeric\] or \[date\] field with name \[field_doesnt_exist\] in any of the indices matching the index pattern/ + catch: /Could not find a \[numeric\] or \[date,date_nanos\] field with name \[field_doesnt_exist\] in any of the indices matching the index pattern/ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser rollup.put_job: