diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 10bab9b3fce92..b44248cf74335 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -125,3 +125,7 @@ ${path.logs} # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting. # #opensearch.experimental.feature.pluggable.caching.enabled: false +# +# Gates the functionality of star tree index, which improves the performance of search aggregations. +# +#opensearch.experimental.feature.composite_index.enabled: true diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 16edec112f123..59e78605c33cb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1324,6 +1324,8 @@ private static void updateIndexMappingsAndBuildSortOrder( // at this point. The validation will take place later in the process // (when all shards are copied in a single place). indexService.getIndexSortSupplier().get(); + // validate composite index fields + indexService.getCompositeIndexConfigSupplier().get(); } if (request.dataStreamName() != null) { MetadataCreateDataStreamService.validateTimestampFieldMapping(mapperService); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 09f32884e0ae1..abd015de2a9c8 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -744,8 +744,11 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS, + SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, - SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING + + // Composite index setting + IndicesService.COMPOSITE_INDEX_ENABLED_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 238df1bd90113..095d02abb3d2d 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -37,6 +37,7 @@ protected FeatureFlagSettings( FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, FeatureFlags.PLUGGABLE_CACHE_SETTING, + FeatureFlags.COMPOSITE_INDEX_SETTING, FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 6fe8dec9c21b1..814e50b99ae24 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -51,6 +51,7 @@ import org.opensearch.index.SearchSlowLog; import org.opensearch.index.TieredMergePolicyProvider; import org.opensearch.index.cache.bitset.BitsetFilterCache; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.FieldMapper; @@ -238,6 +239,14 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for concurrent segment search IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING, IndexSettings.ALLOW_DERIVED_FIELDS, + + // Settings for composite index defaults + CompositeIndexConfig.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, + CompositeIndexConfig.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, + CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING, + CompositeIndexConfig.DEFAULT_METRICS_LIST, + CompositeIndexConfig.DEFAULT_DATE_INTERVALS, + // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { Map groups = s.getAsGroups(); @@ -249,8 +258,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings { } } }, Property.IndexScope), // this allows similarity settings to be passed - Setting.groupSetting("index.analysis.", Property.IndexScope) // this allows analysis settings to be passed - + Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed + Setting.groupSetting("index.composite_index.config.", Property.IndexScope) // this allows composite index settings to be + // passed ) ) ); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 82f43921d2d28..8e2495dce9212 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -72,6 +72,12 @@ public class FeatureFlags { */ public static final String REMOTE_PUBLICATION_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.publication.enabled"; + /** + * Gates the functionality of composite index i.e. star tree index, which improves the performance of search + * aggregations. + */ + public static final String COMPOSITE_INDEX = "opensearch.experimental.feature.composite_index.enabled"; + public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_STORE_MIGRATION_EXPERIMENTAL, false, @@ -100,6 +106,8 @@ public class FeatureFlags { Property.NodeScope ); + public static final Setting COMPOSITE_INDEX_SETTING = Setting.boolSetting(COMPOSITE_INDEX, false, Property.NodeScope); + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -108,7 +116,8 @@ public class FeatureFlags { DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, PLUGGABLE_CACHE_SETTING, - REMOTE_PUBLICATION_EXPERIMENTAL_SETTING + REMOTE_PUBLICATION_EXPERIMENTAL_SETTING, + COMPOSITE_INDEX_SETTING ); /** * Should store the settings from opensearch.yml. diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 3c4cb4fd596c1..18122745a4218 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -606,7 +606,8 @@ public IndexService newIndexService( BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + BooleanSupplier isCompositeIndexCreationEnabled ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -665,7 +666,8 @@ public IndexService newIndexService( translogFactorySupplier, clusterDefaultRefreshIntervalSupplier, recoverySettings, - remoteStoreSettings + remoteStoreSettings, + isCompositeIndexCreationEnabled ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index e501d7eff3f81..96081388baae5 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -72,6 +72,7 @@ import org.opensearch.index.cache.IndexCache; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -183,6 +184,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final CircuitBreakerService circuitBreakerService; private final IndexNameExpressionResolver expressionResolver; private final Supplier indexSortSupplier; + private final Supplier compositeIndexConfigSupplier; private final ValuesSourceRegistry valuesSourceRegistry; private final BiFunction translogFactorySupplier; private final Supplier clusterDefaultRefreshIntervalSupplier; @@ -223,7 +225,8 @@ public IndexService( BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + BooleanSupplier isCompositeIndexCreationEnabled ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -261,6 +264,15 @@ public IndexService( } else { this.indexSortSupplier = () -> null; } + + if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) { + // The validation is done right after the merge of the mapping later in the process ( similar to sort ) + this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig() + .validateAndGetCompositeIndexConfig(mapperService::fieldType, isCompositeIndexCreationEnabled); + } else { + this.compositeIndexConfigSupplier = () -> null; + } + indexFieldData.setListener(new FieldDataCacheListener(this)); this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this)); this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool)); @@ -273,6 +285,7 @@ public IndexService( this.bitsetFilterCache = null; this.warmer = null; this.indexCache = null; + this.compositeIndexConfigSupplier = () -> null; } this.shardStoreDeleter = shardStoreDeleter; @@ -385,6 +398,10 @@ public Supplier getIndexSortSupplier() { return indexSortSupplier; } + public Supplier getCompositeIndexConfigSupplier() { + return compositeIndexConfigSupplier; + } + public synchronized void close(final String reason, boolean delete) throws IOException { if (closed.compareAndSet(false, true)) { deleted.compareAndSet(false, delete); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 613e93698d683..d344e9f580e96 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -48,6 +48,7 @@ import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.Index; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.Translog; @@ -760,6 +761,8 @@ public static IndexMergePolicy fromString(String text) { private final LogByteSizeMergePolicyProvider logByteSizeMergePolicyProvider; private final IndexSortConfig indexSortConfig; private final IndexScopedSettings scopedSettings; + + private final CompositeIndexConfig compositeIndexConfig; private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); private final boolean softDeleteEnabled; private volatile long softDeleteRetentionOperations; @@ -985,6 +988,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.tieredMergePolicyProvider = new TieredMergePolicyProvider(logger, this); this.logByteSizeMergePolicyProvider = new LogByteSizeMergePolicyProvider(logger, this); this.indexSortConfig = new IndexSortConfig(this); + this.compositeIndexConfig = new CompositeIndexConfig(this); searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER); defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE); setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING)); @@ -1740,6 +1744,10 @@ public IndexSortConfig getIndexSortConfig() { return indexSortConfig; } + public CompositeIndexConfig getCompositeIndexConfig() { + return compositeIndexConfig; + } + public IndexScopedSettings getScopedSettings() { return scopedSettings; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java new file mode 100644 index 0000000000000..a0b95b0409da0 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; + +/** + * Composite field which contains dimensions, metrics and index mode specific specs + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeField { + private final String name; + private final List dimensionsOrder; + private final List metrics; + private final CompositeFieldSpec compositeFieldSpec; + + public CompositeField(String name, List dimensions, List metrics, CompositeFieldSpec compositeFieldSpec) { + this.name = name; + this.dimensionsOrder = dimensions; + this.metrics = metrics; + this.compositeFieldSpec = compositeFieldSpec; + } + + public String getName() { + return name; + } + + public List getDimensionsOrder() { + return dimensionsOrder; + } + + public List getMetrics() { + return metrics; + } + + public CompositeFieldSpec getSpec() { + return compositeFieldSpec; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java new file mode 100644 index 0000000000000..829084ebf8b1a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * CompositeFieldSpec interface. + * + * @opensearch.experimental + */ + +@ExperimentalApi +public interface CompositeFieldSpec {} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java new file mode 100644 index 0000000000000..a785d7adb6557 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java @@ -0,0 +1,650 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.indices.IndicesService; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.BooleanSupplier; +import java.util.function.Function; + +/** + * Configuration of composite index containing list of composite fields. + * Each composite field contains dimensions, metrics along with composite index (eg: star tree) specific settings. + * Each composite field will generate a composite index in indexing flow. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeIndexConfig { + private static final Set> ALLOWED_DIMENSION_MAPPED_FIELD_TYPES = Set.of( + NumberFieldMapper.NumberFieldType.class, + DateFieldMapper.DateFieldType.class + ); + + private static final Set> ALLOWED_METRIC_MAPPED_FIELD_TYPES = Set.of( + NumberFieldMapper.NumberFieldType.class + ); + + /** + * This setting determines the max number of composite fields that can be part of composite index config. For each + * composite field, we will generate associated composite index. (eg : star tree index per field ) + */ + public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( + "index.composite_index.max_fields", + 1, + 1, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting determines the max number of dimensions that can be part of composite index field. Number of + * dimensions and associated cardinality has direct effect of composite index size and query performance. + */ + public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( + "index.composite_index.field.max_dimensions", + 10, + 2, + 10, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and + * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa + *

+ * We can remove this later or change it to an enum based constant setting. + * + * @opensearch.experimental + */ + public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( + "index.composite_index.startree.default.max_leaf_docs", + 10000, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * Default intervals for date dimension as part of composite fields + */ + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + "index.composite_index.field.default.date_intervals", + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + CompositeIndexConfig::getTimeUnit, + Setting.Property.IndexScope, + Setting.Property.Final + ); + public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( + "index.composite_index.field.default.metrics", + Arrays.asList( + MetricType.AVG.toString(), + MetricType.COUNT.toString(), + MetricType.SUM.toString(), + MetricType.MAX.toString(), + MetricType.MIN.toString() + ), + MetricType::fromTypeName, + Setting.Property.IndexScope, + Setting.Property.Final + ); + private volatile int maxLeafDocs; + private volatile List defaultDateIntervals; + private volatile List defaultMetrics; + private volatile int maxDimensions; + private volatile int maxFields; + private static final String COMPOSITE_INDEX_CONFIG = "index.composite_index.config"; + private static final String DIMENSIONS_ORDER = "dimensions_order"; + private static final String DIMENSIONS_CONFIG = "dimensions_config"; + private static final String METRICS = "metrics"; + private static final String METRICS_CONFIG = "metrics_config"; + private static final String FIELD = "field"; + private static final String TYPE = "type"; + private static final String INDEX_MODE = "index_mode"; + private static final String STAR_TREE_BUILD_MODE = "build_mode"; + private static final String MAX_LEAF_DOCS = "max_leaf_docs"; + private static final String SKIP_STAR_NODE_CREATION_FOR_DIMS = "skip_star_node_creation_for_dimensions"; + private static final String SPEC = "_spec"; + private final List compositeFields = new ArrayList<>(); + private final IndexSettings indexSettings; + + public CompositeIndexConfig(IndexSettings indexSettings) { + this.setMaxLeafDocs(indexSettings.getValue(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); + this.setDefaultDateIntervals(indexSettings.getValue(DEFAULT_DATE_INTERVALS)); + this.setDefaultMetrics(indexSettings.getValue(DEFAULT_METRICS_LIST)); + this.setMaxDimensions(indexSettings.getValue(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); + this.setMaxFields(indexSettings.getValue(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); + final Map compositeIndexSettings = indexSettings.getSettings().getGroups(COMPOSITE_INDEX_CONFIG); + this.indexSettings = indexSettings; + Set fields = compositeIndexSettings.keySet(); + if (!fields.isEmpty()) { + if (!FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING)) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + if (fields.size() > getMaxFields()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] fields", getMaxFields()) + ); + } + } + for (String field : fields) { + compositeFields.add(buildCompositeField(field, compositeIndexSettings.get(field))); + } + } + + /** + * This returns composite field after performing basic validations but doesn't perform field type based validations + * + */ + private CompositeField buildCompositeField(String field, Settings compositeFieldSettings) { + + List dimensions = buildDimensions(field, compositeFieldSettings); + List metrics = buildMetrics(field, compositeFieldSettings); + List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); + IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); + Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); + CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder, dimensions, this); + return new CompositeField(field, dimensions, metrics, compositeFieldSpec); + } + + /** + * Returns dimensions after performing validations + * + */ + private List buildDimensions(String field, Settings compositeFieldSettings) { + List dimensions = new ArrayList<>(); + List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); + if (dimensionsOrder == null || dimensionsOrder.isEmpty()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "dimensions_order is required for composite index field [%s]", field) + ); + } + if (dimensionsOrder.size() < 2) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Atleast two dimensions are required to build composite index field [%s]", field) + ); + } + + if (dimensionsOrder.size() > getMaxDimensions()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", getMaxDimensions()) + ); + } + + Map dimConfig = compositeFieldSettings.getGroups(DIMENSIONS_CONFIG); + + for (String dimension : dimConfig.keySet()) { + if (!dimensionsOrder.contains(dimension)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "dimension [%s] is not present in dimensions_order for composite index field [%s]", + dimension, + field + ) + ); + } + } + + Set uniqueDimensions = new HashSet<>(); + for (String dimension : dimensionsOrder) { + if (!uniqueDimensions.add(dimension)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate dimension [%s] found in dimensions_order for composite index field [%s]", + dimension, + field + ) + ); + } + dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension), this)); + } + return dimensions; + } + + /** + * Dimension factory based on field type + */ + private static class DimensionFactory { + static Dimension create(String dimension, Settings settings, CompositeIndexConfig compositeIndexConfig) { + if (settings == null) { + return new Dimension(dimension); + } + String field = settings.get(FIELD, dimension); + String type = settings.get(TYPE, DimensionType.DEFAULT.getTypeName()); + switch (DimensionType.fromTypeName(type)) { + case DEFAULT: + return new Dimension(field); + case DATE: + return new DateDimension(field, settings, compositeIndexConfig); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension type [%s] in composite index config", type) + ); + } + } + } + + /** + * The type of dimension source fields + * Default fields are of Numeric type + */ + private enum DimensionType { + DEFAULT("default"), + DATE("date"); + + private final String typeName; + + DimensionType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static DimensionType fromTypeName(String typeName) { + for (DimensionType dimensionType : DimensionType.values()) { + if (dimensionType.getTypeName().equalsIgnoreCase(typeName)) { + return dimensionType; + } + } + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension type in composite index config: [%s] ", typeName) + ); + } + } + + /** + * Returns metrics after performing validations + * + */ + private List buildMetrics(String field, Settings compositeFieldSettings) { + List metrics = new ArrayList<>(); + List metricFields = compositeFieldSettings.getAsList(METRICS); + if (metricFields == null || metricFields.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "metrics is required for composite index field [%s]", field)); + } + Map metricsConfig = compositeFieldSettings.getGroups(METRICS_CONFIG); + + for (String metricField : metricsConfig.keySet()) { + if (!metricFields.contains(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "metric field [%s] is not present in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + } + Set uniqueMetricFields = new HashSet<>(); + for (String metricField : metricFields) { + if (!uniqueMetricFields.add(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric field [%s] found in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + Settings metricSettings = metricsConfig.get(metricField); + if (metricSettings == null) { + metrics.add(new Metric(metricField, getDefaultMetrics())); + } else { + String name = metricSettings.get(FIELD, metricField); + List metricsList = metricSettings.getAsList(METRICS); + if (metricsList.isEmpty()) { + metrics.add(new Metric(name, getDefaultMetrics())); + } else { + List metricTypes = new ArrayList<>(); + Set uniqueMetricTypes = new HashSet<>(); + for (String metric : metricsList) { + if (!uniqueMetricTypes.add(metric)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric type [%s] found in metrics for composite index field [%s]", + metric, + field + ) + ); + } + metricTypes.add(MetricType.fromTypeName(metric)); + } + metrics.add(new Metric(name, metricTypes)); + } + } + } + return metrics; + } + + /** + * Composite field spec factory based on index mode + */ + private static class CompositeFieldSpecFactory { + static CompositeFieldSpec create( + IndexMode indexMode, + Settings settings, + List dimensionsOrder, + List dimensions, + CompositeIndexConfig compositeIndexConfig + ) { + if (settings == null) { + return new StarTreeFieldSpec( + compositeIndexConfig.getMaxLeafDocs(), + new ArrayList<>(), + StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP + ); + } + switch (indexMode) { + case STARTREE: + return buildStarTreeFieldSpec(settings, dimensionsOrder, dimensions, compositeIndexConfig); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid index mode [%s] in composite index config", indexMode) + ); + } + } + } + + private static StarTreeFieldSpec buildStarTreeFieldSpec( + Settings settings, + List dimensionsString, + List dimensions, + CompositeIndexConfig compositeIndexConfig + ) { + StarTreeFieldSpec.StarTreeBuildMode buildMode = StarTreeFieldSpec.StarTreeBuildMode.fromTypeName( + settings.get(STAR_TREE_BUILD_MODE, StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP.getTypeName()) + ); + int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, compositeIndexConfig.getMaxLeafDocs()); + if (maxLeafDocs < 1) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid max_leaf_docs [%s] in composite index config", maxLeafDocs) + ); + } + List skipStarNodeCreationInDims = settings.getAsList(SKIP_STAR_NODE_CREATION_FOR_DIMS, new ArrayList<>()); + Set skipListWithMappedFieldNames = new HashSet<>(); + for (String dim : skipStarNodeCreationInDims) { + if (!dimensionsString.contains(dim)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension [%s] in skip_star_node_creation_for_dims", dim) + ); + } + boolean duplicate = !(skipListWithMappedFieldNames.add(dimensions.get(dimensionsString.indexOf(dim)).getField())); + if (duplicate) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "duplicate dimension [%s] found in skipStarNodeCreationInDims", dim) + ); + } + } + return new StarTreeFieldSpec(maxLeafDocs, new ArrayList<>(skipListWithMappedFieldNames), buildMode); + } + + /** + * Enum for index mode of the underlying composite index + * The default and only index supported right now is star tree index + */ + private enum IndexMode { + STARTREE("startree"); + + private final String typeName; + + IndexMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static IndexMode fromTypeName(String typeName) { + for (IndexMode indexType : IndexMode.values()) { + if (indexType.getTypeName().equalsIgnoreCase(typeName)) { + return indexType; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid index mode in composite index config: [%s] ", typeName)); + } + } + + /** + * Returns the composite fields built based on compositeIndexConfig index settings + */ + public List getCompositeFields() { + return compositeFields; + } + + /** + * Returns whether there are any composite fields as part of the compositeIndexConfig + */ + public boolean hasCompositeFields() { + return !compositeFields.isEmpty(); + } + + /** + * Validates the composite fields based on defaults and based on the mappedFieldType + * Updates CompositeIndexConfig with newer, completely updated composite fields. + * + */ + public CompositeIndexConfig validateAndGetCompositeIndexConfig( + Function fieldTypeLookup, + BooleanSupplier isCompositeIndexCreationEnabled + ) { + if (hasCompositeFields() == false) { + return null; + } + if (!isCompositeIndexCreationEnabled.getAsBoolean()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index cannot be created, enable it using [%s] setting", + IndicesService.COMPOSITE_INDEX_ENABLED_SETTING.getKey() + ) + ); + } + for (CompositeField compositeField : compositeFields) { + for (Dimension dimension : compositeField.getDimensionsOrder()) { + validateDimensionField(dimension, fieldTypeLookup, compositeField.getName()); + } + for (Metric metric : compositeField.getMetrics()) { + validateMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); + } + } + return this; + } + + /** + * Checks whether dimension field type is same as the source field type + */ + private boolean isDimensionMappedToFieldType(Dimension dimension, MappedFieldType fieldType) { + if (fieldType instanceof DateFieldMapper.DateFieldType) { + return dimension instanceof DateDimension; + } + return true; + } + + /** + * Validations : + * The dimension field should be one of the source fields of the index + * The dimension fields must be aggregation compatible (doc values + field data supported) + * The dimension fields should be of numberField type / dateField type + * + */ + private void validateDimensionField(Dimension dimension, Function fieldTypeLookup, String compositeFieldName) { + final MappedFieldType ft = fieldTypeLookup.apply(dimension.getField()); + if (ft == null) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "unknown dimension field [%s] as part of composite field [%s]", + dimension.getField(), + compositeFieldName + ) + ); + } + if (!isDimensionMappedToFieldType(dimension, ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "specify field type [%s] for dimension field [%s] as part of of composite field [%s]", + ft.typeName(), + dimension.getField(), + compositeFieldName + ) + ); + } + if (!isAllowedDimensionFieldType(ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index is not supported for the dimension field [%s] with field type [%s] as part of " + + "composite field [%s]", + dimension.getField(), + ft.typeName(), + compositeFieldName + ) + ); + } + // doc values not present / field data not supported + if (!ft.isAggregatable()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", + dimension.getField(), + ft.typeName(), + compositeFieldName + ) + ); + } + } + + /** + * Validations : + * The metric field should be one of the source fields of the index + * The metric fields must be aggregation compatible (doc values + field data supported) + * The metric fields should be of numberField type + * + */ + private void validateMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { + final MappedFieldType ft = fieldTypeLookup.apply(field); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown metric field [%s] as part of composite field [%s]", field, compositeFieldName) + ); + } + if (!isAllowedMetricFieldType(ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index is not supported for the metric field [%s] with field type [%s] as part of " + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + if (!ft.isAggregatable()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the composite index metric field [%s] with field type [%s] as part of " + + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + } + + private static boolean isAllowedDimensionFieldType(MappedFieldType fieldType) { + return ALLOWED_DIMENSION_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); + } + + private static boolean isAllowedMetricFieldType(MappedFieldType fieldType) { + return ALLOWED_METRIC_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); + } + + public static Rounding.DateTimeUnit getTimeUnit(String expression) { + if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); + } + return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + } + + public void setMaxLeafDocs(int maxLeafDocs) { + this.maxLeafDocs = maxLeafDocs; + } + + public void setDefaultDateIntervals(List defaultDateIntervals) { + this.defaultDateIntervals = defaultDateIntervals; + } + + public void setDefaultMetrics(List defaultMetrics) { + this.defaultMetrics = defaultMetrics; + } + + public void setMaxDimensions(int maxDimensions) { + this.maxDimensions = maxDimensions; + } + + public void setMaxFields(int maxFields) { + this.maxFields = maxFields; + } + + public int getMaxDimensions() { + return maxDimensions; + } + + public int getMaxFields() { + return maxFields; + } + + public int getMaxLeafDocs() { + return maxLeafDocs; + } + + public List getDefaultDateIntervals() { + return defaultDateIntervals; + } + + public List getDefaultMetrics() { + return defaultMetrics; + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java new file mode 100644 index 0000000000000..3e88759ec56b9 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; + +/** + * Date dimension class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class DateDimension extends Dimension { + private final List calendarIntervals; + + public DateDimension(String name, Settings settings, CompositeIndexConfig compositeIndexConfig) { + super(name); + List intervalStrings = settings.getAsList("calendar_interval"); + if (intervalStrings == null || intervalStrings.isEmpty()) { + this.calendarIntervals = compositeIndexConfig.getDefaultDateIntervals(); + } else { + this.calendarIntervals = new ArrayList<>(); + for (String interval : intervalStrings) { + this.calendarIntervals.add(CompositeIndexConfig.getTimeUnit(interval)); + } + } + } + + public List getIntervals() { + return calendarIntervals; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java new file mode 100644 index 0000000000000..a18ffcd1df0db --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Composite index dimension base class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Dimension { + private final String field; + + public Dimension(String field) { + this.field = field; + } + + public String getField() { + return field; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java new file mode 100644 index 0000000000000..9467cf1176f7a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; + +/** + * Holds details of metrics field as part of composite field + */ +@ExperimentalApi +public class Metric { + private final String field; + private final List metrics; + + public Metric(String field, List metrics) { + this.field = field; + this.metrics = metrics; + } + + public String getField() { + return field; + } + + public List getMetrics() { + return metrics; + } + + public void setDefaults(CompositeIndexConfig compositeIndexConfig) { + if (metrics.isEmpty()) { + metrics.addAll(compositeIndexConfig.getDefaultMetrics()); + } + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java new file mode 100644 index 0000000000000..9261279fa5957 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Supported metric types for composite index + */ +@ExperimentalApi +public enum MetricType { + COUNT("count"), + AVG("avg"), + SUM("sum"), + MIN("min"), + MAX("max"); + + private final String typeName; + + MetricType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static MetricType fromTypeName(String typeName) { + for (MetricType metric : MetricType.values()) { + if (metric.getTypeName().equalsIgnoreCase(typeName)) { + return metric; + } + } + throw new IllegalArgumentException("Invalid metric type: " + typeName); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java new file mode 100644 index 0000000000000..c3ac54feeb5e4 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Star tree index specific settings for a composite field. + */ +@ExperimentalApi +public class StarTreeFieldSpec implements CompositeFieldSpec { + + private final AtomicInteger maxLeafDocs = new AtomicInteger(); + private final List skipStarNodeCreationInDims; + private final StarTreeBuildMode buildMode; + + public StarTreeFieldSpec(int maxLeafDocs, List skipStarNodeCreationInDims, StarTreeBuildMode buildMode) { + this.maxLeafDocs.set(maxLeafDocs); + this.skipStarNodeCreationInDims = skipStarNodeCreationInDims; + this.buildMode = buildMode; + } + + /** + * Star tree build mode using which sorting and aggregations are performed during index creation. + * + * @opensearch.experimental + */ + @ExperimentalApi + public enum StarTreeBuildMode { + ON_HEAP("onheap"), + OFF_HEAP("offheap"); + + private final String typeName; + + StarTreeBuildMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static StarTreeBuildMode fromTypeName(String typeName) { + for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) { + if (starTreeBuildMode.getTypeName().equalsIgnoreCase(typeName)) { + return starTreeBuildMode; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", typeName)); + } + } + + public int maxLeafDocs() { + return maxLeafDocs.get(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java new file mode 100644 index 0000000000000..59f18efec26b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Core classes for handling composite indices. + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex; diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 251be8a990055..e087b0d76977b 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -74,6 +74,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.OpenSearchExecutors; @@ -306,6 +307,25 @@ public class IndicesService extends AbstractLifecycleComponent Property.Final ); + /** + * This cluster level setting determines whether composite index is enabled or not + */ + public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( + "indices.composite_index.enabled", + false, + value -> { + if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** * The node's settings. */ @@ -354,6 +374,7 @@ public class IndicesService extends AbstractLifecycleComponent private final BiFunction translogFactorySupplier; private volatile TimeValue clusterDefaultRefreshInterval; private final SearchRequestStats searchRequestStats; + private volatile boolean compositeIndexCreationEnabled; @Override protected void doStart() { @@ -440,6 +461,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setCompositeIndexCreationEnabled); // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -903,7 +926,8 @@ private synchronized IndexService createIndexService( translogFactorySupplier, this::getClusterDefaultRefreshInterval, this.recoverySettings, - this.remoteStoreSettings + this.remoteStoreSettings, + this::isCompositeIndexCreationEnabled ); } @@ -1896,6 +1920,14 @@ private void setIdFieldDataEnabled(boolean value) { this.idFieldDataEnabled = value; } + private void setCompositeIndexCreationEnabled(boolean value) { + this.compositeIndexCreationEnabled = value; + } + + public boolean isCompositeIndexCreationEnabled() { + return compositeIndexCreationEnabled; + } + private void updateDanglingIndicesInfo(Index index) { assert DiscoveryNode.isDataNode(settings) : "dangling indices information should only be persisted on data nodes"; assert nodeWriteDanglingIndicesInfo : "writing dangling indices info is not enabled"; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 3d6a54055d3d5..d5269252681a7 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -1118,6 +1118,7 @@ private IndicesService mockIndicesServices(DocumentMapper documentMapper) throws when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() { }); when(indexService.getIndexSortSupplier()).thenReturn(() -> null); + when(indexService.getCompositeIndexConfigSupplier()).thenReturn(() -> null); // noinspection unchecked return ((CheckedFunction) invocationOnMock.getArguments()[1]).apply(indexService); }); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 4ce4936c047d9..829e65569bc75 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -264,7 +264,8 @@ private IndexService newIndexService(IndexModule module) throws IOException { translogFactorySupplier, () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + () -> false ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java new file mode 100644 index 0000000000000..6145be8887c85 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java @@ -0,0 +1,236 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexService; +import org.opensearch.indices.IndicesService; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class CompositeIndexConfigIntegTests extends OpenSearchIntegTestCase { + + private static final XContentBuilder TEST_MAPPING = createTestMapping(); + + private static XContentBuilder createTestMapping() { + try { + return jsonBuilder().startObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric") + .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") + .field("doc_values", true) + .endObject() + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.COMPOSITE_INDEX, "true").build(); + } + + @Before + public final void setupNodeSettings() { + Settings request = Settings.builder().put(IndicesService.COMPOSITE_INDEX_ENABLED_SETTING.getKey(), true).build(); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get()); + } + + @After + public final void cleanupNodeSettings() { + assertAcked( + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("*")) + .setTransientSettings(Settings.builder().putNull("*")) + ); + } + + public void testInvalidCompositeIndex() { + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("specify field type [date] for dimension field [timestamp] as part of of composite field [my_field]", ex.getMessage()); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "Aggregations not supported for the composite index metric field [numeric] with field type [integer] as part of composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("invalid", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("unknown dimension field [invalid] as part of composite field [my_field]", ex.getMessage()); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("invalid")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("unknown metric field [invalid] as part of composite field [my_field]", ex.getMessage()); + + FeatureFlagSetter.set(FeatureFlags.COMPOSITE_INDEX); + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "2") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get(); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("test"); + GetSettingsResponse indexSettings = client().admin().indices().getSettings(getSettingsRequest).actionGet(); + indexSettings.getIndexToSettings().get("test"); + final Index index = resolveIndex("test"); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + CompositeIndexConfig compositeIndexConfig = indexService.getIndexSettings().getCompositeIndexConfig(); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("timestamp", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("numeric_dv", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0) instanceof DateDimension); + } + } + } + + public void testValidCompositeIndex() { + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "2") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get(); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("test"); + GetSettingsResponse indexSettings = client().admin().indices().getSettings(getSettingsRequest).actionGet(); + indexSettings.getIndexToSettings().get("test"); + final Index index = resolveIndex("test"); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + CompositeIndexConfig compositeIndexConfig = indexService.getIndexSettings().getCompositeIndexConfig(); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("timestamp", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("numeric_dv", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0) instanceof DateDimension); + List expectedMetrics = Arrays.asList( + MetricType.AVG, + MetricType.COUNT, + MetricType.SUM, + MetricType.MAX, + MetricType.MIN + ); + assertEquals(expectedMetrics, compositeIndexConfig.getCompositeFields().get(0).getMetrics().get(0).getMetrics()); + List expectedIntervals = Arrays.asList( + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY + ); + assertEquals( + expectedIntervals, + ((DateDimension) compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0)).getIntervals() + ); + } + } + + } + +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java new file mode 100644 index 0000000000000..ef87345df2017 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java @@ -0,0 +1,627 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.IpFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BooleanSupplier; +import java.util.function.Function; + +import static org.opensearch.common.util.FeatureFlags.COMPOSITE_INDEX; +import static org.opensearch.index.IndexSettingsTests.newIndexMeta; + +/** + * Composite index config settings unit tests + */ +public class CompositeIndexConfigSettingsTests extends OpenSearchTestCase { + + public void testDefaultSettings() { + Settings settings = Settings.EMPTY; + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertFalse(compositeIndexConfig.hasCompositeFields()); + } + + public void testMinimumMetrics() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .build(); + ; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); + assertEquals("metrics is required for composite index field [my_field]", exception.getMessage()); + + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testMinimumDimensions() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); + assertEquals("Atleast two dimensions are required to build composite index field [my_field]", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionType() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); + assertEquals("Invalid dimension type in composite index config: [invalid] ", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidIndexMode() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .put("index.composite_index.config.my_field.index_mode", "invalid") + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); + assertEquals("Invalid index mode in composite index config: [invalid] ", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDefaultNumberofCompositeFieldsValidation() { + Settings settings = Settings.builder() + .put("index.composite_index.max_fields", 1) + .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.field1.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.field2.dimensions_order", Arrays.asList("dim3", "dim4")) + .put("index.composite_index.config.field2.dimensions_config.dim3.field", "dim3_field") + .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") + .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); + assertEquals("composite index can have atmost [1] fields", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testValidCompositeIndexConfig() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .put("index.composite_index.config.my_field.index_mode", "startree") + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("dim1_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("dim2_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1) instanceof DateDimension); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testCompositeIndexMultipleFields() { + Settings settings = Settings.builder() + .put("index.composite_index.max_fields", 2) + .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.field1.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.field2.dimensions_order", Arrays.asList("dim3", "dim4")) + .put("index.composite_index.config.field2.dimensions_config.dim3.field", "dim3_field") + .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") + .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + Exception ex = expectThrows(IllegalArgumentException.class, () -> getEnabledAndMultiFieldIndexSettings(settings)); + assertEquals("Failed to parse value [2] for setting [index.composite_index.max_fields] must be <= 1", ex.getMessage()); + /** + * // uncomment once we add support for multiple fields + IndexSettings indexSettings = getEnabledAndMultiFieldIndexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(2, compositeIndexConfig.getCompositeFields().size()); + assertEquals(2, compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().size()); + assertEquals(2, compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().size()); + assertEquals(1, compositeIndexConfig.getCompositeFields().get(0).getMetrics().size()); + assertEquals(1, compositeIndexConfig.getCompositeFields().get(1).getMetrics().size()); + assertEquals("dim1_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("dim2_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertEquals("dim3_field", compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().get(0).getField()); + assertEquals("dim4_field", compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().get(1).getField()); + assertEquals("metric1", compositeIndexConfig.getCompositeFields().get(0).getMetrics().get(0).getField()); + assertEquals("metric2", compositeIndexConfig.getCompositeFields().get(1).getMetrics().get(0).getField()); + **/ + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testCompositeIndexDateIntervalsSetting() { + Settings settings = Settings.builder() + .putList("index.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "date") + .putList("index.composite_index.config.my_field.dimensions_config.dim1.calendar_interval", Arrays.asList("day", "week")) + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + List expectedIntervals = Arrays.asList( + Rounding.DateTimeUnit.DAY_OF_MONTH, + Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR + ); + assertEquals(expectedIntervals, ((DateDimension) compositeField.getDimensionsOrder().get(0)).getIntervals()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testCompositeIndexMetricsSetting() { + Settings settings = Settings.builder() + .putList("index.composite_index.field.default.metrics", Arrays.asList("count", "max")) + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.my_field.metrics_config.metric1.metrics", Arrays.asList("count", "max")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + List expectedMetrics = Arrays.asList(MetricType.COUNT, MetricType.MAX); + assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testCompositeIndexEnabledSetting() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "default") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.LONG)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric2", NumberFieldMapper.NumberType.LONG)); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); + + Function fieldTypeLookup = fieldTypes::get; + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getDisabledSupplier()) + ); + assertEquals( + "composite index cannot be created, enable it using [indices.composite_index.enabled] setting", + exception.getMessage() + ); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testEnabledWithFFOff() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new CompositeIndexConfig(getEnabledIndexSettings(settings)) + ); + assertEquals( + "star tree index is under an experimental feature and can be activated only by enabling " + + "opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + exception.getMessage() + ); + } + + public void testUnknownDimField() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + + Map fieldTypes = new HashMap<>(); + + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals("unknown dimension field [dim1_field] as part of composite field [my_field]", exception.getMessage()); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testUnknownMetricField() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals("unknown metric field [metric2] as part of composite field [my_field]", exception.getMessage()); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionMappedType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new IpFieldMapper.IpFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "composite index is not supported for the dimension field [dim1_field] with field type [ip] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDimsWithNoDocValues() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put( + "dim1_field", + new NumberFieldMapper.NumberFieldType( + "dim1_field", + NumberFieldMapper.NumberType.LONG, + false, + false, + false, + true, + null, + Collections.emptyMap() + ) + ); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "Aggregations not supported for the dimension field [dim1_field] with field type [long] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testMetricsWithNoDocValues() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put( + "metric1", + new NumberFieldMapper.NumberFieldType( + "metric1", + NumberFieldMapper.NumberType.LONG, + false, + false, + false, + true, + null, + Collections.emptyMap() + ) + ); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "Aggregations not supported for the composite index metric field [metric1] with field type [long] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionMappedKeywordType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new KeywordFieldMapper.KeywordFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "composite index is not supported for the dimension field [dim1_field] with " + + "field type [keyword] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidMetricMappedKeywordType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new DateFieldMapper.DateFieldType("metric1")); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "composite index is not supported for the metric field [metric1] with field type [date] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDefaults() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + CompositeIndexConfig config = compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()); + + assertTrue(config.hasCompositeFields()); + assertEquals(1, config.getCompositeFields().size()); + CompositeField compositeField = config.getCompositeFields().get(0); + List expectedMetrics = Arrays.asList(MetricType.AVG, MetricType.COUNT, MetricType.SUM, MetricType.MAX, MetricType.MIN); + assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + StarTreeFieldSpec spec = (StarTreeFieldSpec) compositeField.getSpec(); + assertEquals(10000, spec.maxLeafDocs()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDimTypeValidation() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "specify field type [date] for dimension field [dim2_field] as part of of composite field [my_field]", + ex.getMessage() + ); + } + + private IndexSettings createIndexSettings(Settings settings) { + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); + } + + private Settings getEnabledSettings() { + return Settings.builder().put("index.composite_index.enabled", true).build(); + } + + public IndexSettings getEnabledIndexSettings(Settings settings) { + Settings enabledSettings = Settings.builder().put(settings).put("index.composite_index.enabled", true).build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", enabledSettings), getEnabledSettings()); + return indexSettings; + } + + public IndexSettings getEnabledAndMultiFieldIndexSettings(Settings settings) { + Settings multiFieldEnabledSettings = Settings.builder() + .put(settings) + .put("index.composite_index.enabled", true) + .put(CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING.getKey(), 2) + .build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", multiFieldEnabledSettings), multiFieldEnabledSettings); + return indexSettings; + } + + private BooleanSupplier getEnabledSupplier() { + return new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + return true; + } + }; + } + + private BooleanSupplier getDisabledSupplier() { + return new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + return false; + } + }; + } +} diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 8f1d58cf201e9..a59e061330b91 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -218,6 +218,7 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() { }); when(indexService.getIndexSortSupplier()).thenReturn(() -> null); + when(indexService.getCompositeIndexConfigSupplier()).thenReturn(() -> null); // noinspection unchecked return ((CheckedFunction) invocationOnMock.getArguments()[1]).apply(indexService); });