From c45d95265144cbbd0483dec835e298b3f6b74d8d Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 12 Aug 2024 12:09:28 -0700 Subject: [PATCH] LogsDB QA tests - add mapping parameters generation (#111772) --- ...ogsIndexModeRandomDataChallengeRestIT.java | 80 ++++++---- .../matchers/source/FieldSpecificMatcher.java | 151 +++++++++++++++++- .../qa/matchers/source/SourceMatcher.java | 8 +- .../datageneration/datasource/DataSource.java | 1 + .../datasource/DataSourceHandler.java | 12 +- .../datasource/DataSourceRequest.java | 25 ++- .../datasource/DataSourceResponse.java | 7 +- .../DefaultMappingParametersHandler.java | 82 ++++++++++ .../DefaultPrimitiveTypesHandler.java | 10 +- .../GenericSubObjectFieldDataGenerator.java | 28 ++-- .../fields/NestedFieldDataGenerator.java | 14 ++ .../fields/ObjectFieldDataGenerator.java | 14 ++ .../fields/leaf/ByteFieldDataGenerator.java | 18 ++- .../fields/leaf/DoubleFieldDataGenerator.java | 18 ++- .../fields/leaf/FloatFieldDataGenerator.java | 18 ++- .../leaf/HalfFloatFieldDataGenerator.java | 18 ++- .../leaf/IntegerFieldDataGenerator.java | 18 ++- .../leaf/KeywordFieldDataGenerator.java | 18 ++- .../fields/leaf/LongFieldDataGenerator.java | 18 ++- .../leaf/ScaledFloatFieldDataGenerator.java | 22 ++- .../fields/leaf/ShortFieldDataGenerator.java | 18 ++- .../leaf/UnsignedLongFieldDataGenerator.java | 18 ++- .../DataGeneratorSnapshotTests.java | 45 +++++- 23 files changed, 563 insertions(+), 98 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index 7add0a82572e3..0b41d62f6fe2c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -17,11 +17,13 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.logsdb.datageneration.fields.PredefinedField; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; import java.time.Instant; +import java.util.HashMap; import java.util.List; /** @@ -30,39 +32,57 @@ */ public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends StandardVersusLogsIndexModeChallengeRestIT { private final boolean fullyDynamicMapping; + private final boolean subobjectsDisabled; private final DataGenerator dataGenerator; public StandardVersusLogsIndexModeRandomDataChallengeRestIT() { super(); this.fullyDynamicMapping = randomBoolean(); + this.subobjectsDisabled = randomBoolean(); - this.dataGenerator = new DataGenerator( - DataGeneratorSpecification.builder() - // Nested fields don't work with subobjects: false. - .withNestedFieldsLimit(0) - .withDataSourceHandlers(List.of(new DataSourceHandler() { - // TODO enable scaled_float fields - // There a difference in synthetic source (precision loss) - // specific to this fields which matcher can't handle. - @Override - public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { - // Unsigned long is not used with dynamic mapping - // since it can initially look like long - // but later fail to parse once big values arrive. - // Double is not used since it maps to float with dynamic mapping - // resulting in precision loss compared to original source. - var excluded = fullyDynamicMapping - ? List.of(FieldType.DOUBLE, FieldType.SCALED_FLOAT, FieldType.UNSIGNED_LONG) - : List.of(FieldType.SCALED_FLOAT); - return new DataSourceResponse.FieldTypeGenerator( - () -> randomValueOtherThanMany(excluded::contains, () -> randomFrom(FieldType.values())) - ); + var specificationBuilder = DataGeneratorSpecification.builder(); + // TODO enable nested fields when subobjects are enabled + // It currently hits a bug with empty nested objects + // Nested fields don't work with subobjects: false. + specificationBuilder = specificationBuilder.withNestedFieldsLimit(0); + this.dataGenerator = new DataGenerator(specificationBuilder.withDataSourceHandlers(List.of(new DataSourceHandler() { + @Override + public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { + // Unsigned long is not used with dynamic mapping + // since it can initially look like long + // but later fail to parse once big values arrive. + // Double is not used since it maps to float with dynamic mapping + // resulting in precision loss compared to original source. + var excluded = fullyDynamicMapping ? List.of(FieldType.DOUBLE, FieldType.SCALED_FLOAT, FieldType.UNSIGNED_LONG) : List.of(); + return new DataSourceResponse.FieldTypeGenerator( + () -> randomValueOtherThanMany(excluded::contains, () -> randomFrom(FieldType.values())) + ); + } + + public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { + if (subobjectsDisabled == false) { + // Use default behavior + return null; + } + + assert request.isNested() == false; + + // "enabled: false" is not compatible with subobjects: false + // "runtime: false/strict/runtime" is not compatible with subobjects: false + return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { + var parameters = new HashMap(); + if (ESTestCase.randomBoolean()) { + parameters.put("dynamic", "true"); } - })) - .withPredefinedFields(List.of(new PredefinedField("host.name", FieldType.KEYWORD))) - .build() - ); + if (ESTestCase.randomBoolean()) { + parameters.put("enabled", "true"); + } + + return parameters; + }); + } + })).withPredefinedFields(List.of(new PredefinedField("host.name", FieldType.KEYWORD))).build()); } @Override @@ -87,10 +107,16 @@ public void baselineMappings(XContentBuilder builder) throws IOException { @Override public void contenderMappings(XContentBuilder builder) throws IOException { if (fullyDynamicMapping == false) { - dataGenerator.writeMapping(builder, b -> builder.field("subobjects", false)); + if (subobjectsDisabled) { + dataGenerator.writeMapping(builder, b -> builder.field("subobjects", false)); + } else { + dataGenerator.writeMapping(builder); + } } else { builder.startObject(); - builder.field("subobjects", false); + if (subobjectsDisabled) { + builder.field("subobjects", false); + } builder.endObject(); } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java index 417fd54fe2d32..10b1922e1e217 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java @@ -13,7 +13,9 @@ import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult; import org.elasticsearch.xcontent.XContentBuilder; +import java.math.BigInteger; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Function; @@ -23,7 +25,7 @@ import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections; interface FieldSpecificMatcher { - MatchResult match(List actual, List expected); + MatchResult match(List actual, List expected, Map actualMapping, Map expectedMapping); class HalfFloatMatcher implements FieldSpecificMatcher { private final XContentBuilder actualMappings; @@ -44,7 +46,12 @@ class HalfFloatMatcher implements FieldSpecificMatcher { } @Override - public MatchResult match(List actual, List expected) { + public MatchResult match( + List actual, + List expected, + Map actualMapping, + Map expectedMapping + ) { var actualHalfFloatBytes = normalize(actual); var expectedHalfFloatBytes = normalize(expected); @@ -76,4 +83,144 @@ private static Set normalize(List values) { .collect(Collectors.toSet()); } } + + class ScaledFloatMatcher implements FieldSpecificMatcher { + private final XContentBuilder actualMappings; + private final Settings.Builder actualSettings; + private final XContentBuilder expectedMappings; + private final Settings.Builder expectedSettings; + + ScaledFloatMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + this.actualMappings = actualMappings; + this.actualSettings = actualSettings; + this.expectedMappings = expectedMappings; + this.expectedSettings = expectedSettings; + } + + @Override + public MatchResult match( + List actual, + List expected, + Map actualMapping, + Map expectedMapping + ) { + var scalingFactor = actualMapping.get("scaling_factor"); + var expectedScalingFactor = expectedMapping.get("scaling_factor"); + if (Objects.equals(scalingFactor, expectedScalingFactor) == false) { + throw new IllegalStateException("Scaling factor for scaled_float field does not match between actual and expected mapping"); + } + + assert scalingFactor instanceof Number; + var expectedNormalized = normalizeExpected(expected, ((Number) scalingFactor).doubleValue()); + var actualNormalized = normalizeActual(actual); + + return actualNormalized.equals(expectedNormalized) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Values of type [scaled_float] don't match after normalization, normalized " + + prettyPrintCollections(actualNormalized, expectedNormalized) + ) + ); + } + + private static Set normalizeExpected(List values, double scalingFactor) { + if (values == null) { + return Set.of(); + } + + return values.stream() + .filter(Objects::nonNull) + .map(ScaledFloatMatcher::toDouble) + // Based on logic in ScaledFloatFieldMapper + .map(v -> { + var encoded = Math.round(v * scalingFactor); + return encoded / scalingFactor; + }) + .collect(Collectors.toSet()); + } + + private static Set normalizeActual(List values) { + if (values == null) { + return Set.of(); + } + + return values.stream().filter(Objects::nonNull).map(ScaledFloatMatcher::toDouble).collect(Collectors.toSet()); + } + + private static double toDouble(Object value) { + return ((Number) value).doubleValue(); + } + } + + class UnsignedLongMatcher implements FieldSpecificMatcher { + private final XContentBuilder actualMappings; + private final Settings.Builder actualSettings; + private final XContentBuilder expectedMappings; + private final Settings.Builder expectedSettings; + + UnsignedLongMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + this.actualMappings = actualMappings; + this.actualSettings = actualSettings; + this.expectedMappings = expectedMappings; + this.expectedSettings = expectedSettings; + } + + @Override + public MatchResult match( + List actual, + List expected, + Map actualMapping, + Map expectedMapping + ) { + var expectedNormalized = normalize(expected); + var actualNormalized = normalize(actual); + + return actualNormalized.equals(expectedNormalized) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Values of type [scaled_float] don't match after normalization, normalized " + + prettyPrintCollections(actualNormalized, expectedNormalized) + ) + ); + } + + private static Set normalize(List values) { + if (values == null) { + return Set.of(); + } + + return values.stream().filter(Objects::nonNull).map(UnsignedLongMatcher::toBigInteger).collect(Collectors.toSet()); + } + + private static BigInteger toBigInteger(Object value) { + if (value instanceof String s) { + return new BigInteger(s, 10); + } + if (value instanceof Long l) { + return BigInteger.valueOf(l); + } + + return (BigInteger) value; + } + } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java index 076135d992a67..f0e188a17631f 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java @@ -54,7 +54,11 @@ public SourceMatcher( this.fieldSpecificMatchers = Map.of( "half_float", - new FieldSpecificMatcher.HalfFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) + new FieldSpecificMatcher.HalfFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings), + "scaled_float", + new FieldSpecificMatcher.ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings), + "unsigned_long", + new FieldSpecificMatcher.UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) ); } @@ -158,7 +162,7 @@ private Optional matchWithFieldSpecificMatcher(String fieldName, Li return Optional.empty(); } - MatchResult matched = fieldSpecificMatcher.match(actualValues, expectedValues); + MatchResult matched = fieldSpecificMatcher.match(actualValues, expectedValues, expectedFieldMapping, actualFieldMapping); return Optional.of(matched); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java index f53b8169f6b70..d550b1ce757a1 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java @@ -34,6 +34,7 @@ public DataSource(Collection additionalHandlers) { this.handlers.add(new DefaultPrimitiveTypesHandler()); this.handlers.add(new DefaultWrappersHandler()); this.handlers.add(new DefaultObjectGenerationHandler()); + this.handlers.add(new DefaultMappingParametersHandler()); } public T get(DataSourceRequest request) { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index 1ee587159ee5f..0cb6ff33c612b 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -33,10 +33,6 @@ default DataSourceResponse.DoubleGenerator handle(DataSourceRequest.DoubleGenera return null; } - default DataSourceResponse.DoubleInRangeGenerator handle(DataSourceRequest.DoubleInRangeGenerator request) { - return null; - } - default DataSourceResponse.FloatGenerator handle(DataSourceRequest.FloatGenerator request) { return null; } @@ -68,4 +64,12 @@ default DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldType default DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectArrayGenerator request) { return null; } + + default DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + return null; + } + + default DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { + return null; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index d28ce7033578c..df3adc458829e 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.logsdb.datageneration.datasource; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; +import org.elasticsearch.logsdb.datageneration.FieldType; public interface DataSourceRequest { TResponse accept(DataSourceHandler handler); @@ -49,14 +50,6 @@ public DataSourceResponse.DoubleGenerator accept(DataSourceHandler handler) { } } - record DoubleInRangeGenerator(double minExclusive, double maxExclusive) - implements - DataSourceRequest { - public DataSourceResponse.DoubleInRangeGenerator accept(DataSourceHandler handler) { - return handler.handle(this); - } - } - record FloatGenerator() implements DataSourceRequest { public DataSourceResponse.FloatGenerator accept(DataSourceHandler handler) { return handler.handle(this); @@ -106,4 +99,20 @@ public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler) return handler.handle(this); } } + + record LeafMappingParametersGenerator(String fieldName, FieldType fieldType) + implements + DataSourceRequest { + public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + + record ObjectMappingParametersGenerator(boolean isNested) + implements + DataSourceRequest { + public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index 867bb9603ca00..2386c4c32ab6c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -10,6 +10,7 @@ import org.elasticsearch.logsdb.datageneration.FieldType; +import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; @@ -27,8 +28,6 @@ record ByteGenerator(Supplier generator) implements DataSourceResponse {} record DoubleGenerator(Supplier generator) implements DataSourceResponse {} - record DoubleInRangeGenerator(Supplier generator) implements DataSourceResponse {} - record FloatGenerator(Supplier generator) implements DataSourceResponse {} record HalfFloatGenerator(Supplier generator) implements DataSourceResponse {} @@ -52,4 +51,8 @@ interface ChildFieldGenerator extends DataSourceResponse { record FieldTypeGenerator(Supplier generator) implements DataSourceResponse {} record ObjectArrayGenerator(Supplier> lengthGenerator) implements DataSourceResponse {} + + record LeafMappingParametersGenerator(Supplier> mappingGenerator) implements DataSourceResponse {} + + record ObjectMappingParametersGenerator(Supplier> mappingGenerator) implements DataSourceResponse {} } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java new file mode 100644 index 0000000000000..aeb34ad2e7049 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.datasource; + +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +public class DefaultMappingParametersHandler implements DataSourceHandler { + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) { + case KEYWORD -> keywordMapping(); + case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT -> numberMapping(); + case UNSIGNED_LONG -> unsignedLongMapping(); + case SCALED_FLOAT -> scaledFloatMapping(); + }); + } + + // TODO enable doc_values: false + // It is disabled because it hits a bug in synthetic source. + private Supplier> keywordMapping() { + return () -> Map.of("store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean()); + } + + // TODO enable doc_values: false + // It is disabled because it hits a bug in synthetic source. + private Supplier> numberMapping() { + return () -> Map.of("store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean()); + } + + private Supplier> unsignedLongMapping() { + return () -> Map.of("store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean()); + } + + private Supplier> scaledFloatMapping() { + return () -> { + var scalingFactor = ESTestCase.randomFrom(10, 1000, 100000, 100.5); + return Map.of("scaling_factor", scalingFactor, "store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean()); + }; + } + + @Override + public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { + if (request.isNested()) { + return new DataSourceResponse.ObjectMappingParametersGenerator( + // TODO enable "false" and "strict" + // It is disabled because it hits a bug in synthetic source. + () -> { + var parameters = new HashMap(); + if (ESTestCase.randomBoolean()) { + parameters.put("dynamic", "true"); + } + + return parameters; + } + ); + } + + // TODO enable "enabled: false" and "dynamic: false/runtime" + // It is disabled because it hits a bug in synthetic source. + return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { + var parameters = new HashMap(); + if (ESTestCase.randomBoolean()) { + parameters.put("dynamic", ESTestCase.randomFrom("true", "strict")); + } + if (ESTestCase.randomBoolean()) { + parameters.put("enabled", "true"); + } + + return parameters; + }); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java index 6838eedc4f6ea..68bb628cc8b27 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java @@ -20,7 +20,8 @@ public DataSourceResponse.LongGenerator handle(DataSourceRequest.LongGenerator r @Override public DataSourceResponse.UnsignedLongGenerator handle(DataSourceRequest.UnsignedLongGenerator request) { - return new DataSourceResponse.UnsignedLongGenerator(() -> new BigInteger(64, ESTestCase.random())); + // TODO there is currently an issue with handling BigInteger in some synthetic source scenarios + return new DataSourceResponse.UnsignedLongGenerator(() -> new BigInteger(64, ESTestCase.random()).toString()); } @Override @@ -43,13 +44,6 @@ public DataSourceResponse.DoubleGenerator handle(DataSourceRequest.DoubleGenerat return new DataSourceResponse.DoubleGenerator(ESTestCase::randomDouble); } - @Override - public DataSourceResponse.DoubleInRangeGenerator handle(DataSourceRequest.DoubleInRangeGenerator request) { - return new DataSourceResponse.DoubleInRangeGenerator( - () -> ESTestCase.randomDoubleBetween(request.minExclusive(), request.maxExclusive(), false) - ); - } - @Override public DataSourceResponse.FloatGenerator handle(DataSourceRequest.FloatGenerator request) { return new DataSourceResponse.FloatGenerator(ESTestCase::randomFloat); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java index 1a3da3b63add0..8a6a8939c7ddb 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java @@ -49,9 +49,9 @@ List generateChildFields() { var fieldName = generateFieldName(existingFieldNames); if (context.shouldAddObjectField()) { - result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject()))); + result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject()), false)); } else if (context.shouldAddNestedField()) { - result.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject()))); + result.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject()), false)); } else { var fieldType = context.fieldTypeGenerator().generator().get(); result.add(leafField(fieldType, fieldName)); @@ -103,19 +103,19 @@ static void writeChildFieldsData(XContentBuilder document, Iterable private ChildField leafField(FieldType type, String fieldName) { var generator = switch (type) { - case KEYWORD -> new KeywordFieldDataGenerator(context.specification().dataSource()); - case LONG -> new LongFieldDataGenerator(context.specification().dataSource()); - case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(context.specification().dataSource()); - case INTEGER -> new IntegerFieldDataGenerator(context.specification().dataSource()); - case SHORT -> new ShortFieldDataGenerator(context.specification().dataSource()); - case BYTE -> new ByteFieldDataGenerator(context.specification().dataSource()); - case DOUBLE -> new DoubleFieldDataGenerator(context.specification().dataSource()); - case FLOAT -> new FloatFieldDataGenerator(context.specification().dataSource()); - case HALF_FLOAT -> new HalfFloatFieldDataGenerator(context.specification().dataSource()); - case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(context.specification().dataSource()); + case KEYWORD -> new KeywordFieldDataGenerator(fieldName, context.specification().dataSource()); + case LONG -> new LongFieldDataGenerator(fieldName, context.specification().dataSource()); + case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, context.specification().dataSource()); + case INTEGER -> new IntegerFieldDataGenerator(fieldName, context.specification().dataSource()); + case SHORT -> new ShortFieldDataGenerator(fieldName, context.specification().dataSource()); + case BYTE -> new ByteFieldDataGenerator(fieldName, context.specification().dataSource()); + case DOUBLE -> new DoubleFieldDataGenerator(fieldName, context.specification().dataSource()); + case FLOAT -> new FloatFieldDataGenerator(fieldName, context.specification().dataSource()); + case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, context.specification().dataSource()); + case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, context.specification().dataSource()); }; - return new ChildField(fieldName, generator); + return new ChildField(fieldName, generator, false); } private String generateFieldName(Set existingFields) { @@ -128,5 +128,5 @@ private String generateFieldName(Set existingFields) { return fieldName; } - record ChildField(String fieldName, FieldDataGenerator generator) {} + record ChildField(String fieldName, FieldDataGenerator generator, boolean dynamic) {} } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java index f52b739418034..3ba220a64d4fd 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java @@ -10,17 +10,27 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.List; +import java.util.Map; public class NestedFieldDataGenerator implements FieldDataGenerator { private final Context context; + private final Map mappingParameters; private final List childFields; NestedFieldDataGenerator(Context context) { this.context = context; + + this.mappingParameters = context.specification() + .dataSource() + .get(new DataSourceRequest.ObjectMappingParametersGenerator(true)) + .mappingGenerator() + .get(); + var genericGenerator = new GenericSubObjectFieldDataGenerator(context); this.childFields = genericGenerator.generateChildFields(); } @@ -32,6 +42,10 @@ public CheckedConsumer mappingWriter() { b.field("type", "nested"); + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + b.startObject("properties"); GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, childFields); b.endObject(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java index 522bb2b1772b0..00f2977c8af33 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java @@ -10,17 +10,27 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.List; +import java.util.Map; public class ObjectFieldDataGenerator implements FieldDataGenerator { private final Context context; + private final Map mappingParameters; private final List childFields; ObjectFieldDataGenerator(Context context) { this.context = context; + + this.mappingParameters = context.specification() + .dataSource() + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false)) + .mappingGenerator() + .get(); + var genericGenerator = new GenericSubObjectFieldDataGenerator(context); this.childFields = genericGenerator.generateChildFields(); } @@ -30,6 +40,10 @@ public CheckedConsumer mappingWriter() { return b -> { b.startObject(); + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + b.startObject("properties"); GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, childFields); b.endObject(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java index 07a7bd65b67fb..d7461294456e9 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class ByteFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public ByteFieldDataGenerator(DataSource dataSource) { + public ByteFieldDataGenerator(String fieldName, DataSource dataSource) { var bytes = dataSource.get(new DataSourceRequest.ByteGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> bytes.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.BYTE)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "byte").endObject(); + return b -> { + b.startObject().field("type", "byte"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java index 84c5afe2fae51..98d588754ef93 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class DoubleFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public DoubleFieldDataGenerator(DataSource dataSource) { + public DoubleFieldDataGenerator(String fieldName, DataSource dataSource) { var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.DOUBLE)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "double").endObject(); + return b -> { + b.startObject().field("type", "double"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java index 34e401a99bd0a..438617ee3c38d 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class FloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public FloatFieldDataGenerator(DataSource dataSource) { + public FloatFieldDataGenerator(String fieldName, DataSource dataSource) { var floats = dataSource.get(new DataSourceRequest.FloatGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> floats.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.FLOAT)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "float").endObject(); + return b -> { + b.startObject().field("type", "float"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java index 3201926e35041..b21467a2afdcc 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class HalfFloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public HalfFloatFieldDataGenerator(DataSource dataSource) { + public HalfFloatFieldDataGenerator(String fieldName, DataSource dataSource) { var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> halfFloats.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.HALF_FLOAT)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "half_float").endObject(); + return b -> { + b.startObject().field("type", "half_float"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java index a532d77abc80e..06f5f12fceac3 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class IntegerFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public IntegerFieldDataGenerator(DataSource dataSource) { + public IntegerFieldDataGenerator(String fieldName, DataSource dataSource) { var ints = dataSource.get(new DataSourceRequest.IntegerGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> ints.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.INTEGER)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "integer").endObject(); + return b -> { + b.startObject().field("type", "integer"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java index 913cd5657dc6f..a33b4ce1b2365 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class KeywordFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public KeywordFieldDataGenerator(DataSource dataSource) { + public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) { var strings = dataSource.get(new DataSourceRequest.StringGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> strings.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.KEYWORD)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "keyword").endObject(); + return b -> { + b.startObject().field("type", "keyword"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java index 3627385f51a7c..6f6fc5db253aa 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class LongFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public LongFieldDataGenerator(DataSource dataSource) { + public LongFieldDataGenerator(String fieldName, DataSource dataSource) { var longs = dataSource.get(new DataSourceRequest.LongGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> longs.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.LONG)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "long").endObject(); + return b -> { + b.startObject().field("type", "long"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java index 38fa0504cf7e7..fb50933f74d70 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java @@ -10,31 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class ScaledFloatFieldDataGenerator implements FieldDataGenerator { - private final double scalingFactor; private final Supplier valueGenerator; + private final Map mappingParameters; - public ScaledFloatFieldDataGenerator(DataSource dataSource) { - var positiveDoubles = dataSource.get(new DataSourceRequest.DoubleInRangeGenerator(0, Double.MAX_VALUE)); - this.scalingFactor = positiveDoubles.generator().get(); - + public ScaledFloatFieldDataGenerator(String fieldName, DataSource dataSource) { var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.SCALED_FLOAT)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "scaled_float").field("scaling_factor", scalingFactor).endObject(); + return b -> { + b.startObject().field("type", "scaled_float"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java index 511b31794a925..43d390c0857b9 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class ShortFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public ShortFieldDataGenerator(DataSource dataSource) { + public ShortFieldDataGenerator(String fieldName, DataSource dataSource) { var shorts = dataSource.get(new DataSourceRequest.ShortGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> shorts.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.SHORT)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "short").endObject(); + return b -> { + b.startObject().field("type", "short"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java index 327b3260fdec5..7e833d6486e52 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java @@ -10,27 +10,41 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; public class UnsignedLongFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Map mappingParameters; - public UnsignedLongFieldDataGenerator(DataSource dataSource) { + public UnsignedLongFieldDataGenerator(String fieldName, DataSource dataSource) { var unsignedLongs = dataSource.get(new DataSourceRequest.UnsignedLongGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> unsignedLongs.generator().get()); + this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.UNSIGNED_LONG)) + .mappingGenerator() + .get(); } @Override public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "unsigned_long").endObject(); + return b -> { + b.startObject().field("type", "unsigned_long"); + + for (var entry : mappingParameters.entrySet()) { + b.field(entry.getKey(), entry.getValue()); + } + + b.endObject(); + }; } @Override diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java index 6c1b0c22f305d..8ff5998a31d45 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentType; import java.util.List; +import java.util.Map; import java.util.Optional; public class DataGeneratorSnapshotTests extends ESTestCase { @@ -40,24 +41,31 @@ public void testSnapshot() throws Exception { "_doc" : { "properties" : { "f1" : { + "dynamic" : "false", "properties" : { "f2" : { + "dynamic" : "false", "properties" : { "f3" : { - "type" : "keyword" + "type" : "keyword", + "store" : "true" }, "f4" : { - "type" : "long" + "type" : "long", + "index" : "false" } } }, "f5" : { + "dynamic" : "false", "properties" : { "f6" : { - "type" : "keyword" + "type" : "keyword", + "store" : "true" }, "f7" : { - "type" : "long" + "type" : "long", + "index" : "false" } } } @@ -65,20 +73,25 @@ public void testSnapshot() throws Exception { }, "f8" : { "type" : "nested", + "dynamic" : "false", "properties" : { "f9" : { "type" : "nested", + "dynamic" : "false", "properties" : { "f10" : { - "type" : "keyword" + "type" : "keyword", + "store" : "true" }, "f11" : { - "type" : "long" + "type" : "long", + "index" : "false" } } }, "f12" : { - "type" : "keyword" + "type" : "keyword", + "store" : "true" } } } @@ -199,6 +212,24 @@ public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeG return FieldType.LONG; }); } + + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + if (request.fieldType() == FieldType.KEYWORD) { + return new DataSourceResponse.LeafMappingParametersGenerator(() -> Map.of("store", "true")); + } + + if (request.fieldType() == FieldType.LONG) { + return new DataSourceResponse.LeafMappingParametersGenerator(() -> Map.of("index", "false")); + } + + return null; + } + + @Override + public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { + return new DataSourceResponse.ObjectMappingParametersGenerator(() -> Map.of("dynamic", "false")); + } } private static class StaticChildFieldGenerator implements DataSourceResponse.ChildFieldGenerator {