diff --git a/docs/changelog/111457.yaml b/docs/changelog/111457.yaml new file mode 100644 index 0000000000000..f4ad4ee53eb0a --- /dev/null +++ b/docs/changelog/111457.yaml @@ -0,0 +1,6 @@ +pr: 111457 +summary: Add support for boolean dimensions +area: TSDB +type: enhancement +issues: + - 111338 diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index de89fa1ca3f31..01573658c33d0 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -107,6 +107,7 @@ parameter: * <> * <> * <> +* <> For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. For details refer to <>. diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc index e081b122355bb..32f3d13edf581 100644 --- a/docs/reference/mapping/types/boolean.asciidoc +++ b/docs/reference/mapping/types/boolean.asciidoc @@ -223,6 +223,13 @@ The following parameters are accepted by `boolean` fields: Metadata about the field. +`time_series_dimension`:: +(Optional, Boolean) ++ +-- +include::keyword.asciidoc[tag=dimension] +-- + [[boolean-synthetic-source]] ==== Synthetic `_source` diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 9113abaa1d249..d20231a6d6cf2 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -232,13 +232,13 @@ dynamic templates: refresh: true body: - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5", "attributes.another.dim3": true }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5", "attributes.another.dim3": true }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5", "attributes.another.dim3": false }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5", "attributes.another.dim3": false }' - do: search: @@ -264,7 +264,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } + - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -283,7 +283,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } + - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -302,7 +302,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } + - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -321,7 +321,25 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } + - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + another.dim3: true + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -554,13 +572,13 @@ dynamic templates with nesting: refresh: true body: - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.a.much.deeper.nested.dim": "AC" }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.another.dim3": true, "attributes.a.much.deeper.nested.dim": "AC" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.a.much.deeper.nested.dim": "AC" }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.another.dim3": true, "attributes.a.much.deeper.nested.dim": "AC" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.a.much.deeper.nested.dim": "BD" }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.another.dim3": false, "attributes.a.much.deeper.nested.dim": "BD" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.a.much.deeper.nested.dim": "BD" }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.another.dim3": false, "attributes.a.much.deeper.nested.dim": "BD" }' - do: search: @@ -586,7 +604,7 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" } + - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -605,7 +623,7 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" } + - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -624,7 +642,7 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" } + - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -643,7 +661,26 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" } + - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + another.dim3: true + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -662,7 +699,7 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" } + - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -763,6 +800,19 @@ dynamic templates with incremental indexing: - '{ "@timestamp": "2023-09-01T13:06:10.138Z","data": "330", "attributes.a.much.deeper.nested.dim": "BD" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - '{ "@timestamp": "2023-09-01T13:06:10.238Z","data": "340", "attributes.a.much.deeper.nested.dim": "BD" }' + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:05:08.138Z","data": "210", "resource.attributes.another.deeper.dim3": true }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:05:09.138Z","data": "220", "resource.attributes.another.deeper.dim3": true }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:05:10.138Z","data": "230", "resource.attributes.another.deeper.dim3": false }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:05:10.238Z","data": "240", "resource.attributes.another.deeper.dim3": false }' - do: search: @@ -770,7 +820,7 @@ dynamic templates with incremental indexing: body: size: 0 - - match: { hits.total.value: 16 } + - match: { hits.total.value: 20 } - do: search: @@ -862,6 +912,24 @@ dynamic templates with incremental indexing: - length: { aggregations.filterA.tsids.buckets: 1 } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + another.deeper.dim3: true + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + --- subobject in passthrough object auto flatten: - requires: @@ -1056,7 +1124,7 @@ dimensions with ignore_malformed and ignore_above: --- non string dimension fields: - requires: - cluster_features: ["mapper.pass_through_priority", "routing.boolean_routing_path"] + cluster_features: ["mapper.pass_through_priority", "routing.boolean_routing_path", "mapper.boolean_dimension"] reason: support for priority in passthrough objects - do: allowed_warnings: @@ -1098,13 +1166,6 @@ non string dimension fields: match_mapping_type: string mapping: type: keyword - # ES doesn't support boolean fields as dimensions, yet - # however, support has been added to have boolean fields in the routing path - # as long as these boolean fields are converted to a field type that supports dimensions - - booleans_as_keywords: - match_mapping_type: boolean - mapping: - type: keyword - double_as_double: match_mapping_type: double mapping: @@ -1145,7 +1206,7 @@ non string dimension fields: fields: [ "*" ] - match: { hits.total.value: 1 } - match: { hits.hits.0.fields.attributes\.string: [ "foo" ] } - - match: { hits.hits.0.fields.attributes\.boolean: [ "true" ] } + - match: { hits.hits.0.fields.attributes\.boolean: [ true ] } - match: { hits.hits.0.fields.attributes\.integer: [ 1 ] } - match: { hits.hits.0.fields.attributes\.double: [ 1.1 ] } - match: { hits.hits.0.fields.attributes\.host\.ip: [ "127.0.0.1" ] } @@ -1160,7 +1221,12 @@ non string dimension fields: index: $idx0name expand_wildcards: hidden - match: { .$idx0name.mappings.properties.attributes.properties.string.type: 'keyword' } - - match: { .$idx0name.mappings.properties.attributes.properties.boolean.type: 'keyword' } + - match: { .$idx0name.mappings.properties.attributes.properties.string.time_series_dimension: true } + - match: { .$idx0name.mappings.properties.attributes.properties.boolean.type: 'boolean' } + - match: { .$idx0name.mappings.properties.attributes.properties.boolean.time_series_dimension: true } - match: { .$idx0name.mappings.properties.attributes.properties.integer.type: 'long' } + - match: { .$idx0name.mappings.properties.attributes.properties.integer.time_series_dimension: true } - match: { .$idx0name.mappings.properties.attributes.properties.double.type: 'double' } + - match: { .$idx0name.mappings.properties.attributes.properties.double.time_series_dimension: true } - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' } + - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.time_series_dimension: true } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index f0cc51f3effa5..1f0088ec96478 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -60,6 +61,8 @@ public class BooleanFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "boolean"; + static final NodeFeature BOOLEAN_DIMENSION = new NodeFeature("mapper.boolean_dimension"); + public static class Values { public static final BytesRef TRUE = new BytesRef("T"); public static final BytesRef FALSE = new BytesRef("F"); @@ -69,7 +72,7 @@ private static BooleanFieldMapper toType(FieldMapper in) { return (BooleanFieldMapper) in; } - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); @@ -94,6 +97,8 @@ public static final class Builder extends FieldMapper.Builder { private final IndexVersion indexCreatedVersion; + private final Parameter dimension; + public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) { super(name); this.scriptCompiler = Objects.requireNonNull(scriptCompiler); @@ -106,15 +111,36 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform ); this.script.precludesParameters(ignoreMalformed, nullValue); addScriptValidation(script, indexed, docValues); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> { + if (v && (indexed.getValue() == false || docValues.getValue() == false)) { + throw new IllegalArgumentException( + "Field [" + + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + + "] requires that [" + + indexed.name + + "] and [" + + docValues.name + + "] are true" + ); + } + }); + } + + public Builder dimension(boolean dimension) { + this.dimension.setValue(dimension); + return this; } @Override protected Parameter[] getParameters() { - return new Parameter[] { meta, docValues, indexed, nullValue, stored, script, onScriptError, ignoreMalformed }; + return new Parameter[] { meta, docValues, indexed, nullValue, stored, script, onScriptError, ignoreMalformed, dimension }; } @Override public BooleanFieldMapper build(MapperBuilderContext context) { + if (inheritDimensionParameterFromParentObject(context)) { + dimension(true); + } MappedFieldType ft = new BooleanFieldType( context.buildFullName(leafName()), indexed.getValue() && indexCreatedVersion.isLegacyIndexVersion() == false, @@ -122,7 +148,8 @@ public BooleanFieldMapper build(MapperBuilderContext context) { docValues.getValue(), nullValue.getValue(), scriptValues(), - meta.getValue() + meta.getValue(), + dimension.getValue() ); return new BooleanFieldMapper( leafName(), @@ -158,6 +185,7 @@ public static final class BooleanFieldType extends TermBasedFieldType { private final Boolean nullValue; private final FieldValues scriptValues; + private final boolean isDimension; public BooleanFieldType( String name, @@ -166,11 +194,13 @@ public BooleanFieldType( boolean hasDocValues, Boolean nullValue, FieldValues scriptValues, - Map meta + Map meta, + boolean isDimension ) { super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.nullValue = nullValue; this.scriptValues = scriptValues; + this.isDimension = isDimension; } public BooleanFieldType(String name) { @@ -182,7 +212,7 @@ public BooleanFieldType(String name, boolean isIndexed) { } public BooleanFieldType(String name, boolean isIndexed, boolean hasDocValues) { - this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap()); + this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false); } @Override @@ -195,6 +225,11 @@ public boolean isSearchable() { return isIndexed() || hasDocValues(); } + @Override + public boolean isDimension() { + return isDimension; + } + @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { if (format != null) { @@ -455,6 +490,10 @@ private void indexValue(DocumentParserContext context, Boolean value) { if (value == null) { return; } + + if (fieldType().isDimension()) { + context.getDimensions().addBoolean(fieldType().name(), value).validate(context.indexSettings()); + } if (indexed) { context.doc().add(new StringField(fieldType().name(), value ? Values.TRUE : Values.FALSE, Field.Store.NO)); } @@ -480,7 +519,17 @@ protected void indexScriptValues( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this); + return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(fieldType().isDimension()) + .init(this); + } + + @Override + public void doValidate(MappingLookup lookup) { + if (fieldType().isDimension() && null != lookup.nestedLookup().getNestedParent(fullPath())) { + throw new IllegalArgumentException( + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + " can't be configured in nested field [" + fullPath() + "]" + ); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java index 3cf90f2385525..aa69e4db50e76 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java @@ -44,6 +44,8 @@ default DocumentDimensions addString(String fieldName, String value) { DocumentDimensions addUnsignedLong(String fieldName, long value); + DocumentDimensions addBoolean(String fieldName, boolean value); + DocumentDimensions validate(IndexSettings settings); /** @@ -83,6 +85,12 @@ public DocumentDimensions addUnsignedLong(String fieldName, long value) { return this; } + @Override + public DocumentDimensions addBoolean(String fieldName, boolean value) { + add(fieldName); + return this; + } + @Override public DocumentDimensions validate(final IndexSettings settings) { // DO NOTHING diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index ac3275c5e7119..15d77ba6d2229 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -30,7 +30,8 @@ public Set getFeatures() { DocumentMapper.INDEX_SORTING_ON_NESTED, KeywordFieldMapper.KEYWORD_DIMENSION_IGNORE_ABOVE, IndexModeFieldMapper.QUERYING_INDEX_MODE, - NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS + NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS, + BooleanFieldMapper.BOOLEAN_DIMENSION ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 112b3ec96b39e..f1c6a072c2d9e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -344,6 +344,18 @@ public DocumentDimensions addUnsignedLong(String fieldName, long value) { } } + @Override + public DocumentDimensions addBoolean(String fieldName, boolean value) { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.write((byte) 'b'); + out.write(value ? 't' : 'f'); + add(fieldName, out.bytes()); + } catch (IOException e) { + throw new IllegalArgumentException("Dimension field cannot be serialized.", e); + } + return this; + } + @Override public DocumentDimensions validate(final IndexSettings settings) { if (settings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ID_HASHING) @@ -415,6 +427,8 @@ public static Map decodeTsidAsMap(StreamInput in) { } case (byte) 'd' -> // parse a double result.put(name, in.readDouble()); + case (byte) 'b' -> // parse a boolean + result.put(name, in.read() == 't'); default -> throw new IllegalArgumentException("Cannot parse [" + name + "]: Unknown type [" + type + "]"); } } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 1289095ae8d2c..28e65a6803ae3 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -358,7 +358,9 @@ public void testRequireDocValuesOnDoubles() { public void testRequireDocValuesOnBools() { doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field")); - doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap())); + doTestRequireDocValues( + new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false) + ); } public void testFieldDataCacheExpire() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 70e2fee7a003a..e08a443bd74cb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -13,8 +13,12 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.script.BooleanFieldScript; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -24,6 +28,7 @@ import java.util.List; import java.util.function.Function; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class BooleanFieldMapperTests extends MapperTestCase { @@ -44,6 +49,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("index", b -> b.field("index", false)); checker.registerConflictCheck("store", b -> b.field("store", true)); checker.registerConflictCheck("null_value", b -> b.field("null_value", true)); + + registerDimensionChecks(checker); } public void testExistsQueryDocValuesDisabled() throws IOException { @@ -207,6 +214,74 @@ public void testScriptAndPrecludedParameters() { assertThat(e.getMessage(), equalTo("Failed to parse mapping: Field [null_value] cannot be set in conjunction with field [script]")); } + public void testDimension() throws IOException { + // Test default setting + MapperService mapperService = createMapperService(fieldMapping(b -> minimalMapping(b))); + BooleanFieldMapper.BooleanFieldType ft = (BooleanFieldMapper.BooleanFieldType) mapperService.fieldType("field"); + assertFalse(ft.isDimension()); + + assertDimension(true, BooleanFieldMapper.BooleanFieldType::isDimension); + assertDimension(false, BooleanFieldMapper.BooleanFieldType::isDimension); + } + + public void testDimensionIndexedAndDocvalues() { + { + Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true).field("index", false).field("doc_values", false); + }))); + assertThat( + e.getCause().getMessage(), + containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + ); + } + { + Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true).field("index", true).field("doc_values", false); + }))); + assertThat( + e.getCause().getMessage(), + containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + ); + } + { + Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true).field("index", false).field("doc_values", true); + }))); + assertThat( + e.getCause().getMessage(), + containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + ); + } + } + + public void testDimensionMultiValuedField() throws IOException { + XContentBuilder mapping = fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true); + }); + DocumentMapper mapper = randomBoolean() ? createDocumentMapper(mapping) : createTimeSeriesModeDocumentMapper(mapping); + + Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", true, false)))); + assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + } + + public void testDimensionInRoutingPath() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("time_series_dimension", true))); + IndexSettings settings = createIndexSettings( + IndexVersion.current(), + Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .build() + ); + mapper.documentMapper().validate(settings, false); // Doesn't throw + } + @Override protected List exampleMalformedValues() { return List.of(exampleMalformedValue("a").errorMatches("Failed to parse value [a] as only [true] or [false] are allowed.")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java index 55f076e985c4d..0466525288d7b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java @@ -79,7 +79,8 @@ public void testFetchSourceValue() throws IOException { true, true, null, - Collections.emptyMap() + Collections.emptyMap(), + false ); assertEquals(List.of(true), fetchSourceValue(nullFieldType, null)); }