LogsDB QA tests - add mapping parameters generation (elastic#111772)

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java

@@ -17,11 +17,13 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
 import org.elasticsearch.logsdb.datageneration.fields.PredefinedField;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.HashMap;
 import java.util.List;
 
 /**
@@ -30,39 +32,57 @@
  */
 public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends StandardVersusLogsIndexModeChallengeRestIT {
     private final boolean fullyDynamicMapping;
+    private final boolean subobjectsDisabled;
 
     private final DataGenerator dataGenerator;
 
     public StandardVersusLogsIndexModeRandomDataChallengeRestIT() {
         super();
         this.fullyDynamicMapping = randomBoolean();
+        this.subobjectsDisabled = randomBoolean();
 
-        this.dataGenerator = new DataGenerator(
-            DataGeneratorSpecification.builder()
-                // Nested fields don't work with subobjects: false.
-                .withNestedFieldsLimit(0)
-                .withDataSourceHandlers(List.of(new DataSourceHandler() {
-                    // TODO enable scaled_float fields
-                    // There a difference in synthetic source (precision loss)
-                    // specific to this fields which matcher can't handle.
-                    @Override
-                    public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
-                        // Unsigned long is not used with dynamic mapping
-                        // since it can initially look like long
-                        // but later fail to parse once big values arrive.
-                        // Double is not used since it maps to float with dynamic mapping
-                        // resulting in precision loss compared to original source.
-                        var excluded = fullyDynamicMapping
-                            ? List.of(FieldType.DOUBLE, FieldType.SCALED_FLOAT, FieldType.UNSIGNED_LONG)
-                            : List.of(FieldType.SCALED_FLOAT);
-                        return new DataSourceResponse.FieldTypeGenerator(
-                            () -> randomValueOtherThanMany(excluded::contains, () -> randomFrom(FieldType.values()))
-                        );
+        var specificationBuilder = DataGeneratorSpecification.builder();
+        // TODO enable nested fields when subobjects are enabled
+        // It currently hits a bug with empty nested objects
+        // Nested fields don't work with subobjects: false.
+        specificationBuilder = specificationBuilder.withNestedFieldsLimit(0);
+        this.dataGenerator = new DataGenerator(specificationBuilder.withDataSourceHandlers(List.of(new DataSourceHandler() {
+            @Override
+            public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
+                // Unsigned long is not used with dynamic mapping
+                // since it can initially look like long
+                // but later fail to parse once big values arrive.
+                // Double is not used since it maps to float with dynamic mapping
+                // resulting in precision loss compared to original source.
+                var excluded = fullyDynamicMapping ? List.of(FieldType.DOUBLE, FieldType.SCALED_FLOAT, FieldType.UNSIGNED_LONG) : List.of();
+                return new DataSourceResponse.FieldTypeGenerator(
+                    () -> randomValueOtherThanMany(excluded::contains, () -> randomFrom(FieldType.values()))
+                );
+            }
+
+            public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
+                if (subobjectsDisabled == false) {
+                    // Use default behavior
+                    return null;
+                }
+
+                assert request.isNested() == false;
+
+                // "enabled: false" is not compatible with subobjects: false
+                // "runtime: false/strict/runtime" is not compatible with subobjects: false
+                return new DataSourceResponse.ObjectMappingParametersGenerator(() -> {
+                    var parameters = new HashMap<String, Object>();
+                    if (ESTestCase.randomBoolean()) {
+                        parameters.put("dynamic", "true");
                     }
-                }))
-                .withPredefinedFields(List.of(new PredefinedField("host.name", FieldType.KEYWORD)))
-                .build()
-        );
+                    if (ESTestCase.randomBoolean()) {
+                        parameters.put("enabled", "true");
+                    }
+
+                    return parameters;
+                });
+            }
+        })).withPredefinedFields(List.of(new PredefinedField("host.name", FieldType.KEYWORD))).build());
     }
 
     @Override
@@ -87,10 +107,16 @@ public void baselineMappings(XContentBuilder builder) throws IOException {
     @Override
     public void contenderMappings(XContentBuilder builder) throws IOException {
         if (fullyDynamicMapping == false) {
-            dataGenerator.writeMapping(builder, b -> builder.field("subobjects", false));
+            if (subobjectsDisabled) {
+                dataGenerator.writeMapping(builder, b -> builder.field("subobjects", false));
+            } else {
+                dataGenerator.writeMapping(builder);
+            }
         } else {
             builder.startObject();
-            builder.field("subobjects", false);
+            if (subobjectsDisabled) {
+                builder.field("subobjects", false);
+            }
             builder.endObject();
         }
     }

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java

@@ -13,7 +13,9 @@
 import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
 import org.elasticsearch.xcontent.XContentBuilder;
 
+import java.math.BigInteger;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
@@ -23,7 +25,7 @@
 import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
 
 interface FieldSpecificMatcher {
-    MatchResult match(List<Object> actual, List<Object> expected);
+    MatchResult match(List<Object> actual, List<Object> expected, Map<String, Object> actualMapping, Map<String, Object> expectedMapping);
 
     class HalfFloatMatcher implements FieldSpecificMatcher {
         private final XContentBuilder actualMappings;
@@ -44,7 +46,12 @@ class HalfFloatMatcher implements FieldSpecificMatcher {
         }
 
         @Override
-        public MatchResult match(List<Object> actual, List<Object> expected) {
+        public MatchResult match(
+            List<Object> actual,
+            List<Object> expected,
+            Map<String, Object> actualMapping,
+            Map<String, Object> expectedMapping
+        ) {
             var actualHalfFloatBytes = normalize(actual);
             var expectedHalfFloatBytes = normalize(expected);
 
@@ -76,4 +83,144 @@ private static Set<Short> normalize(List<Object> values) {
                 .collect(Collectors.toSet());
         }
     }
+
+    class ScaledFloatMatcher implements FieldSpecificMatcher {
+        private final XContentBuilder actualMappings;
+        private final Settings.Builder actualSettings;
+        private final XContentBuilder expectedMappings;
+        private final Settings.Builder expectedSettings;
+
+        ScaledFloatMatcher(
+            XContentBuilder actualMappings,
+            Settings.Builder actualSettings,
+            XContentBuilder expectedMappings,
+            Settings.Builder expectedSettings
+        ) {
+            this.actualMappings = actualMappings;
+            this.actualSettings = actualSettings;
+            this.expectedMappings = expectedMappings;
+            this.expectedSettings = expectedSettings;
+        }
+
+        @Override
+        public MatchResult match(
+            List<Object> actual,
+            List<Object> expected,
+            Map<String, Object> actualMapping,
+            Map<String, Object> expectedMapping
+        ) {
+            var scalingFactor = actualMapping.get("scaling_factor");
+            var expectedScalingFactor = expectedMapping.get("scaling_factor");
+            if (Objects.equals(scalingFactor, expectedScalingFactor) == false) {
+                throw new IllegalStateException("Scaling factor for scaled_float field does not match between actual and expected mapping");
+            }
+
+            assert scalingFactor instanceof Number;
+            var expectedNormalized = normalizeExpected(expected, ((Number) scalingFactor).doubleValue());
+            var actualNormalized = normalizeActual(actual);
+
+            return actualNormalized.equals(expectedNormalized)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Values of type [scaled_float] don't match after normalization, normalized "
+                            + prettyPrintCollections(actualNormalized, expectedNormalized)
+                    )
+                );
+        }
+
+        private static Set<Double> normalizeExpected(List<Object> values, double scalingFactor) {
+            if (values == null) {
+                return Set.of();
+            }
+
+            return values.stream()
+                .filter(Objects::nonNull)
+                .map(ScaledFloatMatcher::toDouble)
+                // Based on logic in ScaledFloatFieldMapper
+                .map(v -> {
+                    var encoded = Math.round(v * scalingFactor);
+                    return encoded / scalingFactor;
+                })
+                .collect(Collectors.toSet());
+        }
+
+        private static Set<Double> normalizeActual(List<Object> values) {
+            if (values == null) {
+                return Set.of();
+            }
+
+            return values.stream().filter(Objects::nonNull).map(ScaledFloatMatcher::toDouble).collect(Collectors.toSet());
+        }
+
+        private static double toDouble(Object value) {
+            return ((Number) value).doubleValue();
+        }
+    }
+
+    class UnsignedLongMatcher implements FieldSpecificMatcher {
+        private final XContentBuilder actualMappings;
+        private final Settings.Builder actualSettings;
+        private final XContentBuilder expectedMappings;
+        private final Settings.Builder expectedSettings;
+
+        UnsignedLongMatcher(
+            XContentBuilder actualMappings,
+            Settings.Builder actualSettings,
+            XContentBuilder expectedMappings,
+            Settings.Builder expectedSettings
+        ) {
+            this.actualMappings = actualMappings;
+            this.actualSettings = actualSettings;
+            this.expectedMappings = expectedMappings;
+            this.expectedSettings = expectedSettings;
+        }
+
+        @Override
+        public MatchResult match(
+            List<Object> actual,
+            List<Object> expected,
+            Map<String, Object> actualMapping,
+            Map<String, Object> expectedMapping
+        ) {
+            var expectedNormalized = normalize(expected);
+            var actualNormalized = normalize(actual);
+
+            return actualNormalized.equals(expectedNormalized)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Values of type [scaled_float] don't match after normalization, normalized "
+                            + prettyPrintCollections(actualNormalized, expectedNormalized)
+                    )
+                );
+        }
+
+        private static Set<BigInteger> normalize(List<Object> values) {
+            if (values == null) {
+                return Set.of();
+            }
+
+            return values.stream().filter(Objects::nonNull).map(UnsignedLongMatcher::toBigInteger).collect(Collectors.toSet());
+        }
+
+        private static BigInteger toBigInteger(Object value) {
+            if (value instanceof String s) {
+                return new BigInteger(s, 10);
+            }
+            if (value instanceof Long l) {
+                return BigInteger.valueOf(l);
+            }
+
+            return (BigInteger) value;
+        }
+    }
 }

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java

@@ -54,7 +54,11 @@ public SourceMatcher(
 
         this.fieldSpecificMatchers = Map.of(
             "half_float",
-            new FieldSpecificMatcher.HalfFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)
+            new FieldSpecificMatcher.HalfFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings),
+            "scaled_float",
+            new FieldSpecificMatcher.ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings),
+            "unsigned_long",
+            new FieldSpecificMatcher.UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)
         );
     }
 
@@ -158,7 +162,7 @@ private Optional<MatchResult> matchWithFieldSpecificMatcher(String fieldName, Li
             return Optional.empty();
         }
 
-        MatchResult matched = fieldSpecificMatcher.match(actualValues, expectedValues);
+        MatchResult matched = fieldSpecificMatcher.match(actualValues, expectedValues, expectedFieldMapping, actualFieldMapping);
         return Optional.of(matched);
     }
 

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java

@@ -34,6 +34,7 @@ public DataSource(Collection<DataSourceHandler> additionalHandlers) {
         this.handlers.add(new DefaultPrimitiveTypesHandler());
         this.handlers.add(new DefaultWrappersHandler());
         this.handlers.add(new DefaultObjectGenerationHandler());
+        this.handlers.add(new DefaultMappingParametersHandler());
     }
 
     public <T extends DataSourceResponse> T get(DataSourceRequest<T> request) {

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java

@@ -33,10 +33,6 @@ default DataSourceResponse.DoubleGenerator handle(DataSourceRequest.DoubleGenera
         return null;
     }
 
-    default DataSourceResponse.DoubleInRangeGenerator handle(DataSourceRequest.DoubleInRangeGenerator request) {
-        return null;
-    }
-
     default DataSourceResponse.FloatGenerator handle(DataSourceRequest.FloatGenerator request) {
         return null;
     }
@@ -68,4 +64,12 @@ default DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldType
     default DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectArrayGenerator request) {
         return null;
     }
+
+    default DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
+        return null;
+    }
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.logsdb.datageneration.datasource;
 
 import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
+import org.elasticsearch.logsdb.datageneration.FieldType;
 
 public interface DataSourceRequest<TResponse extends DataSourceResponse> {
     TResponse accept(DataSourceHandler handler);
@@ -49,14 +50,6 @@ public DataSourceResponse.DoubleGenerator accept(DataSourceHandler handler) {
         }
     }
 
-    record DoubleInRangeGenerator(double minExclusive, double maxExclusive)
-        implements
-            DataSourceRequest<DataSourceResponse.DoubleInRangeGenerator> {
-        public DataSourceResponse.DoubleInRangeGenerator accept(DataSourceHandler handler) {
-            return handler.handle(this);
-        }
-    }
-
     record FloatGenerator() implements DataSourceRequest<DataSourceResponse.FloatGenerator> {
         public DataSourceResponse.FloatGenerator accept(DataSourceHandler handler) {
             return handler.handle(this);
@@ -106,4 +99,20 @@ public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler)
             return handler.handle(this);
         }
     }
+
+    record LeafMappingParametersGenerator(String fieldName, FieldType fieldType)
+        implements
+            DataSourceRequest<DataSourceResponse.LeafMappingParametersGenerator> {
+        public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record ObjectMappingParametersGenerator(boolean isNested)
+        implements
+            DataSourceRequest<DataSourceResponse.ObjectMappingParametersGenerator> {
+        public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
 }