diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index d6bc810355396..f0ac98faffda9 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -85,4 +85,3 @@ ${error.file} # HDFS ForkJoinPool.common() support by SecurityManager -Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory --Dopensearch.experimental.feature.composite_index.star_tree.enabled=true diff --git a/server/src/internalClusterTest/java/org/opensearch/index/codec/CompositeCodecIT.java b/server/src/internalClusterTest/java/org/opensearch/index/codec/CompositeCodecIT.java new file mode 100644 index 0000000000000..f78e8f501430d --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/index/codec/CompositeCodecIT.java @@ -0,0 +1,2 @@ +package org.opensearch.index.codec;public class CompositeCodecIT { +} diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 39deac33a0f06..8e5193b650868 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -19,6 +19,7 @@ import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; @@ -89,6 +90,55 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool } } + private static XContentBuilder createMaxDimTestMapping() { + try { + return jsonBuilder().startObject() + .startObject("composite") + .startObject("startree-1") + .field("type", "star_tree") + .startObject("config") + .startArray("ordered_dimensions") + .startObject() + .field("name", "timestamp") + .startArray("calendar_intervals") + .value("day") + .value("month") + .endArray() + .endObject() + .startObject() + .field("name", "dim2") + .endObject() + .startObject() + .field("name", "dim3") + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", "dim2") + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("dim2") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("dim3") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + private static XContentBuilder createTestMappingWithoutStarTree(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { try { return jsonBuilder().startObject() @@ -329,6 +379,32 @@ public void testInvalidDimCompositeIndex() { ); } + public void testMaxDimsCompositeIndex() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) + .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2)) + .get() + ); + assertEquals( + "Failed to parse mapping [_doc]: ordered_dimensions cannot have more than 2 dimensions for star tree field [startree-1]", + ex.getMessage() + ); + } + + public void testMaxCalendarIntervalsCompositeIndex() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) + .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1)) + .get() + ); + assertEquals( + "Failed to parse mapping [_doc]: At most [1] calendar intervals are allowed in dimension [timestamp]", + ex.getMessage() + ); + } + public void testUnsupportedDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index b10a7d8155056..504bc622ec12e 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -33,6 +33,8 @@ package org.opensearch.index.mapper; import org.opensearch.common.compress.CompressedXContent; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -46,6 +48,7 @@ import java.io.IOException; import java.util.Collection; +import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; import static org.hamcrest.Matchers.containsString; public class ObjectMapperTests extends OpenSearchSingleNodeTestCase { @@ -487,6 +490,76 @@ public void testDerivedFields() throws Exception { assertEquals("date", mapper.typeName()); } + public void testCompositeFields() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("tweet") + .startObject("composite") + .startObject("startree") + .field("type", "star_tree") + .startObject("config") + .startArray("ordered_dimensions") + .startObject() + .field("name", "@timestamp") + .endObject() + .startObject() + .field("name", "status") + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", "status") + .endObject() + .startObject() + .field("name", "metric_field") + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "date") + .endObject() + .startObject("status") + .field("type", "integer") + .endObject() + .startObject("metric_field") + .field("type", "integer") + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> createIndex("invalid").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)) + ); + assertEquals( + "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.star_tree.enabled feature flag in the JVM options", + ex.getMessage() + ); + + final Settings starTreeEnabledSettings = Settings.builder().put(STAR_TREE_INDEX, "true").build(); + FeatureFlags.initializeFeatureFlags(starTreeEnabledSettings); + + DocumentMapper documentMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("tweet", new CompressedXContent(mapping)); + + Mapper mapper = documentMapper.root().getMapper("startree"); + assertTrue(mapper instanceof StarTreeMapper); + StarTreeMapper starTreeMapper = (StarTreeMapper) mapper; + assertEquals("star_tree", starTreeMapper.fieldType().typeName()); + // Check that field in properties was parsed correctly as well + mapper = documentMapper.root().getMapper("@timestamp"); + assertNotNull(mapper); + assertEquals("date", mapper.typeName()); + + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + @Override protected Collection> getPlugins() { return pluginList(InternalSettingsPlugin.class); diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index af72a39f67000..3144b1b007924 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -10,16 +10,24 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Rounding; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.CompositeIndexSettings; +import org.opensearch.index.compositeindex.CompositeIndexValidator; import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.junit.After; import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -207,6 +215,161 @@ public void testInvalidSingleDim() { ); } + public void testMetric() { + List m1 = new ArrayList<>(); + m1.add(MetricStat.MAX); + Metric metric1 = new Metric("name", m1); + Metric metric2 = new Metric("name", m1); + assertEquals(metric1, metric2); + List m2 = new ArrayList<>(); + m2.add(MetricStat.MAX); + m2.add(MetricStat.COUNT); + metric2 = new Metric("name", m2); + assertNotEquals(metric1, metric2); + + assertEquals(MetricStat.COUNT, MetricStat.fromTypeName("count")); + assertEquals(MetricStat.MAX, MetricStat.fromTypeName("max")); + assertEquals(MetricStat.MIN, MetricStat.fromTypeName("min")); + assertEquals(MetricStat.SUM, MetricStat.fromTypeName("sum")); + assertEquals(MetricStat.AVG, MetricStat.fromTypeName("avg")); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> MetricStat.fromTypeName("invalid")); + assertEquals("Invalid metric stat: invalid", ex.getMessage()); + } + + public void testDimensions() { + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + DateDimension d1 = new DateDimension("name", d1CalendarIntervals); + DateDimension d2 = new DateDimension("name", d1CalendarIntervals); + assertEquals(d1, d2); + d2 = new DateDimension("name1", d1CalendarIntervals); + assertNotEquals(d1, d2); + List d2CalendarIntervals = new ArrayList<>(); + d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + d2 = new DateDimension("name", d2CalendarIntervals); + assertNotEquals(d1, d2); + NumericDimension n1 = new NumericDimension("name"); + NumericDimension n2 = new NumericDimension("name"); + assertEquals(n1, n2); + n2 = new NumericDimension("name1"); + assertNotEquals(n1, n2); + } + + public void testStarTreeField() { + List m1 = new ArrayList<>(); + m1.add(MetricStat.MAX); + Metric metric1 = new Metric("name", m1); + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + DateDimension d1 = new DateDimension("name", d1CalendarIntervals); + NumericDimension n1 = new NumericDimension("numeric"); + NumericDimension n2 = new NumericDimension("name1"); + + List metrics = List.of(metric1); + List dims = List.of(d1, n2); + StarTreeFieldConfiguration config = new StarTreeFieldConfiguration( + 100, + Set.of("name"), + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP + ); + + StarTreeField field1 = new StarTreeField("starTree", dims, metrics, config); + StarTreeField field2 = new StarTreeField("starTree", dims, metrics, config); + assertEquals(field1, field2); + + dims = List.of(d1, n2, n1); + field2 = new StarTreeField("starTree", dims, metrics, config); + assertNotEquals(field1, field2); + + dims = List.of(d1, n2); + metrics = List.of(metric1, metric1); + field2 = new StarTreeField("starTree", dims, metrics, config); + assertNotEquals(field1, field2); + + dims = List.of(d1, n2); + metrics = List.of(metric1); + StarTreeFieldConfiguration config1 = new StarTreeFieldConfiguration( + 1000, + Set.of("name"), + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP + ); + field2 = new StarTreeField("starTree", dims, metrics, config1); + assertNotEquals(field1, field2); + + config1 = new StarTreeFieldConfiguration(100, Set.of("name", "field2"), StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP); + field2 = new StarTreeField("starTree", dims, metrics, config1); + assertNotEquals(field1, field2); + + config1 = new StarTreeFieldConfiguration(100, Set.of("name"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP); + field2 = new StarTreeField("starTree", dims, metrics, config1); + assertNotEquals(field1, field2); + + field2 = new StarTreeField("starTree", dims, metrics, config); + assertEquals(field1, field2); + } + + public void testValidations() throws IOException { + MapperService mapperService = createMapperService(getExpandedMapping("status", "size")); + Settings settings = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), true).build(); + CompositeIndexSettings enabledCompositeIndexSettings = new CompositeIndexSettings( + settings, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + CompositeIndexValidator.validate(mapperService, enabledCompositeIndexSettings, mapperService.getIndexSettings()); + settings = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), false).build(); + CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings( + settings, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + MapperService finalMapperService = mapperService; + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> CompositeIndexValidator.validate(finalMapperService, compositeIndexSettings, finalMapperService.getIndexSettings()) + ); + assertEquals( + "star tree index cannot be created, enable it using [indices.composite_index.star_tree.enabled] setting", + ex.getMessage() + ); + + MapperService mapperServiceInvalid = createMapperService(getInvalidMappingWithDv(false, false, false, true)); + ex = expectThrows( + IllegalArgumentException.class, + () -> CompositeIndexValidator.validate( + mapperServiceInvalid, + enabledCompositeIndexSettings, + mapperServiceInvalid.getIndexSettings() + ) + ); + assertEquals( + "Aggregations not supported for the metrics field [metric_field] with field type [integer] as part of star tree field", + ex.getMessage() + ); + + MapperService mapperServiceInvalidDim = createMapperService(getInvalidMappingWithDv(false, false, true, false)); + ex = expectThrows( + IllegalArgumentException.class, + () -> CompositeIndexValidator.validate( + mapperServiceInvalidDim, + enabledCompositeIndexSettings, + mapperServiceInvalidDim.getIndexSettings() + ) + ); + assertEquals( + "Aggregations not supported for the dimension field [@timestamp] with field type [date] as part of star tree field", + ex.getMessage() + ); + + MapperParsingException mapperParsingExceptionex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMappingWith2StarTrees()) + ); + assertEquals( + "Failed to parse mapping [_doc]: Composite fields cannot have more than [1] fields", + mapperParsingExceptionex.getMessage() + ); + } + private XContentBuilder getExpandedMapping(String dim, String metric) throws IOException { return topMapping(b -> { b.startObject("composite"); @@ -310,6 +473,74 @@ private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetric }); } + private XContentBuilder getMinMappingWith2StarTrees() throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "@timestamp"); + b.endObject(); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.endArray(); + + b.startArray("metrics"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.startObject(); + b.field("name", "metric_field"); + b.endObject(); + b.endArray(); + + b.endObject(); + b.endObject(); + + b.startObject("startree1"); + b.field("type", "star_tree"); + b.startObject("config"); + + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "@timestamp"); + b.endObject(); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.endArray(); + + b.startArray("metrics"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.startObject(); + b.field("name", "metric_field"); + b.endObject(); + b.endArray(); + + b.endObject(); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("@timestamp"); + b.field("type", "date"); + b.endObject(); + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + b.startObject("metric_field"); + b.field("type", "integer"); + b.endObject(); + + b.endObject(); + }); + } + private XContentBuilder getInvalidMapping( boolean singleDim, boolean invalidSkipDims, @@ -380,6 +611,74 @@ private XContentBuilder getInvalidMapping( }); } + private XContentBuilder getInvalidMappingWithDv( + boolean singleDim, + boolean invalidSkipDims, + boolean invalidDimType, + boolean invalidMetricType + ) throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + + b.startArray("skip_star_node_creation_for_dimensions"); + { + if (invalidSkipDims) { + b.value("invalid"); + } + b.value("status"); + } + b.endArray(); + b.startArray("ordered_dimensions"); + if (!singleDim) { + b.startObject(); + b.field("name", "@timestamp"); + b.endObject(); + } + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.startObject(); + b.field("name", "metric_field"); + b.endObject(); + b.endArray(); + b.endObject(); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("@timestamp"); + if (!invalidDimType) { + b.field("type", "date"); + b.field("doc_values", "true"); + } else { + b.field("type", "date"); + b.field("doc_values", "false"); + } + b.endObject(); + + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + b.startObject("metric_field"); + if (invalidMetricType) { + b.field("type", "integer"); + b.field("doc_values", "false"); + } else { + b.field("type", "integer"); + b.field("doc_values", "true"); + } + b.endObject(); + b.endObject(); + }); + } + private XContentBuilder getInvalidMapping(boolean singleDim, boolean invalidSkipDims, boolean invalidDimType, boolean invalidMetricType) throws IOException { return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false);