diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b5dd289ec85..df4c2c3bd5480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Store] Add dynamic cluster settings to set timeout for segments upload to Remote Store ([#13679](https://github.com/opensearch-project/OpenSearch/pull/13679)) - [Remote Store] Upload translog checkpoint as object metadata to translog.tlog([#13637](https://github.com/opensearch-project/OpenSearch/pull/13637)) - Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819)) +- [Star Tree Index] Star tree index config changes ([#13917](https://github.com/opensearch-project/OpenSearch/pull/13917)) ### Dependencies - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559)) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 10bab9b3fce92..b44248cf74335 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -125,3 +125,7 @@ ${path.logs} # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting. # #opensearch.experimental.feature.pluggable.caching.enabled: false +# +# Gates the functionality of star tree index, which improves the performance of search aggregations. +# +#opensearch.experimental.feature.composite_index.enabled: true diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 16edec112f123..59e78605c33cb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1324,6 +1324,8 @@ private static void updateIndexMappingsAndBuildSortOrder( // at this point. The validation will take place later in the process // (when all shards are copied in a single place). indexService.getIndexSortSupplier().get(); + // validate composite index fields + indexService.getCompositeIndexConfigSupplier().get(); } if (request.dataStreamName() != null) { MetadataCreateDataStreamService.validateTimestampFieldMapping(mapperService); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index a2be2ea4510e0..7db247119a282 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -113,6 +113,7 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.remote.RemoteStorePressureSettings; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.store.remote.filecache.FileCache; @@ -741,7 +742,15 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS, - RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA + RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, + + // Composite index settings + CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING, + CompositeIndexSettings.COMPOSITE_INDEX_MAX_FIELDS_SETTING, + CompositeIndexSettings.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, + CompositeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, + CompositeIndexSettings.DEFAULT_METRICS_LIST, + CompositeIndexSettings.DEFAULT_DATE_INTERVALS ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 7a364de1c5dc6..2f21d508673c8 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -36,6 +36,7 @@ protected FeatureFlagSettings( FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, - FeatureFlags.PLUGGABLE_CACHE_SETTING + FeatureFlags.PLUGGABLE_CACHE_SETTING, + FeatureFlags.COMPOSITE_INDEX_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 980c432774f6e..9b4521cc3647a 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -249,8 +249,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { } } }, Property.IndexScope), // this allows similarity settings to be passed - Setting.groupSetting("index.analysis.", Property.IndexScope) // this allows analysis settings to be passed - + Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed + Setting.groupSetting("index.composite_index.", Property.IndexScope) // this allows composite index settings to be passed ) ) ); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 62cfbd861d4d9..fe8081d888e98 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -67,6 +67,12 @@ public class FeatureFlags { */ public static final String PLUGGABLE_CACHE = "opensearch.experimental.feature.pluggable.caching.enabled"; + /** + * Gates the functionality of composite index i.e. star tree index, which improves the performance of search + * aggregations. + */ + public static final String COMPOSITE_INDEX = "opensearch.experimental.feature.composite_index.enabled"; + public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_STORE_MIGRATION_EXPERIMENTAL, false, @@ -89,6 +95,8 @@ public class FeatureFlags { public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); + public static final Setting COMPOSITE_INDEX_SETTING = Setting.boolSetting(COMPOSITE_INDEX, false, Property.NodeScope); + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -96,7 +104,8 @@ public class FeatureFlags { TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, - PLUGGABLE_CACHE_SETTING + PLUGGABLE_CACHE_SETTING, + COMPOSITE_INDEX_SETTING ); /** * Should store the settings from opensearch.yml. diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 3c4cb4fd596c1..aaec6bfec2123 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -66,6 +66,7 @@ import org.opensearch.index.cache.query.DisabledQueryCache; import org.opensearch.index.cache.query.IndexQueryCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -606,7 +607,8 @@ public IndexService newIndexService( BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -665,7 +667,8 @@ public IndexService newIndexService( translogFactorySupplier, clusterDefaultRefreshIntervalSupplier, recoverySettings, - remoteStoreSettings + remoteStoreSettings, + compositeIndexSettings ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index e501d7eff3f81..6f994e8f67c5b 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -72,6 +72,8 @@ import org.opensearch.index.cache.IndexCache; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexConfig; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -183,6 +185,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final CircuitBreakerService circuitBreakerService; private final IndexNameExpressionResolver expressionResolver; private final Supplier indexSortSupplier; + private final Supplier compositeIndexConfigSupplier; private final ValuesSourceRegistry valuesSourceRegistry; private final BiFunction translogFactorySupplier; private final Supplier clusterDefaultRefreshIntervalSupplier; @@ -223,7 +226,8 @@ public IndexService( BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -261,6 +265,14 @@ public IndexService( } else { this.indexSortSupplier = () -> null; } + + if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) { + this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig() + .validateAndGetCompositeIndexConfig(mapperService::fieldType, compositeIndexSettings); + } else { + this.compositeIndexConfigSupplier = () -> null; + } + indexFieldData.setListener(new FieldDataCacheListener(this)); this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this)); this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool)); @@ -273,6 +285,7 @@ public IndexService( this.bitsetFilterCache = null; this.warmer = null; this.indexCache = null; + this.compositeIndexConfigSupplier = () -> null; } this.shardStoreDeleter = shardStoreDeleter; @@ -385,6 +398,10 @@ public Supplier getIndexSortSupplier() { return indexSortSupplier; } + public Supplier getCompositeIndexConfigSupplier() { + return compositeIndexConfigSupplier; + } + public synchronized void close(final String reason, boolean delete) throws IOException { if (closed.compareAndSet(false, true)) { deleted.compareAndSet(false, delete); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 2b0a62c18d5a7..4cfdffdce4ee0 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -48,6 +48,7 @@ import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.Index; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.Translog; @@ -753,6 +754,8 @@ public static IndexMergePolicy fromString(String text) { private final LogByteSizeMergePolicyProvider logByteSizeMergePolicyProvider; private final IndexSortConfig indexSortConfig; private final IndexScopedSettings scopedSettings; + + private final CompositeIndexConfig compositeIndexConfig; private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); private final boolean softDeleteEnabled; private volatile long softDeleteRetentionOperations; @@ -966,6 +969,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.tieredMergePolicyProvider = new TieredMergePolicyProvider(logger, this); this.logByteSizeMergePolicyProvider = new LogByteSizeMergePolicyProvider(logger, this); this.indexSortConfig = new IndexSortConfig(this); + this.compositeIndexConfig = new CompositeIndexConfig(this); searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER); defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE); setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING)); @@ -1720,6 +1724,10 @@ public IndexSortConfig getIndexSortConfig() { return indexSortConfig; } + public CompositeIndexConfig getCompositeIndexConfig() { + return compositeIndexConfig; + } + public IndexScopedSettings getScopedSettings() { return scopedSettings; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java new file mode 100644 index 0000000000000..a0b95b0409da0 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; + +/** + * Composite field which contains dimensions, metrics and index mode specific specs + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeField { + private final String name; + private final List dimensionsOrder; + private final List metrics; + private final CompositeFieldSpec compositeFieldSpec; + + public CompositeField(String name, List dimensions, List metrics, CompositeFieldSpec compositeFieldSpec) { + this.name = name; + this.dimensionsOrder = dimensions; + this.metrics = metrics; + this.compositeFieldSpec = compositeFieldSpec; + } + + public String getName() { + return name; + } + + public List getDimensionsOrder() { + return dimensionsOrder; + } + + public List getMetrics() { + return metrics; + } + + public CompositeFieldSpec getSpec() { + return compositeFieldSpec; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java new file mode 100644 index 0000000000000..51a2b454abcd9 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * CompositeFieldSpec interface. + * + * @opensearch.experimental + */ + +@ExperimentalApi +public interface CompositeFieldSpec { + void setDefaults(CompositeIndexSettings compositeIndexSettings); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java new file mode 100644 index 0000000000000..308c4b62d3996 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java @@ -0,0 +1,505 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static org.opensearch.index.compositeindex.CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING; + +/** + * Configuration of composite index containing list of composite fields. + * Each composite field contains dimensions, metrics along with composite index (eg: star tree) specific settings. + * Each composite field will generate a composite index in indexing flow. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeIndexConfig { + private static final Set> ALLOWED_DIMENSION_MAPPED_FIELD_TYPES = Set.of( + NumberFieldMapper.NumberFieldType.class, + DateFieldMapper.DateFieldType.class + ); + + private static final Set> ALLOWED_METRIC_MAPPED_FIELD_TYPES = Set.of( + NumberFieldMapper.NumberFieldType.class + ); + + private static final String COMPOSITE_INDEX_CONFIG = "index.composite_index.config"; + private static final String DIMENSIONS_ORDER = "dimensions_order"; + private static final String DIMENSIONS_CONFIG = "dimensions_config"; + private static final String METRICS = "metrics"; + private static final String METRICS_CONFIG = "metrics_config"; + private static final String FIELD = "field"; + private static final String TYPE = "type"; + private static final String INDEX_MODE = "index_mode"; + private static final String STAR_TREE_BUILD_MODE = "build_mode"; + private static final String MAX_LEAF_DOCS = "max_leaf_docs"; + private static final String SKIP_STAR_NODE_CREATION_FOR_DIMS = "skip_star_node_creation_for_dimensions"; + private static final String SPEC = "_spec"; + private final List compositeFields = new ArrayList<>(); + + public CompositeIndexConfig(IndexSettings indexSettings) { + + final Map compositeIndexSettings = indexSettings.getSettings().getGroups(COMPOSITE_INDEX_CONFIG); + Set fields = compositeIndexSettings.keySet(); + for (String field : fields) { + compositeFields.add(buildCompositeField(field, compositeIndexSettings.get(field))); + } + } + + /** + * This returns composite field after performing basic validations and doesn't do field type validations etc + * + */ + private CompositeField buildCompositeField(String field, Settings compositeFieldSettings) { + List dimensions = new ArrayList<>(); + List metrics = new ArrayList<>(); + List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); + if (dimensionsOrder == null || dimensionsOrder.isEmpty()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "dimensions_order is required for composite index field [%s]", field) + ); + } + if (dimensionsOrder.size() < 2) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Atleast two dimensions are required to build composite index field [%s]", field) + ); + } + + Map dimConfig = compositeFieldSettings.getGroups(DIMENSIONS_CONFIG); + + for (String dimension : dimConfig.keySet()) { + if (!dimensionsOrder.contains(dimension)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "dimension [%s] is not present in dimensions_order for composite index field [%s]", + dimension, + field + ) + ); + } + } + + List metricFields = compositeFieldSettings.getAsList(METRICS); + if (metricFields == null || metricFields.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "metrics is required for composite index field [%s]", field)); + } + Map metricsConfig = compositeFieldSettings.getGroups(METRICS_CONFIG); + + for (String metricField : metricsConfig.keySet()) { + if (!metricFields.contains(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "metric field [%s] is not present in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + } + + Set uniqueDimensions = new HashSet<>(); + for (String dimension : dimensionsOrder) { + if (!uniqueDimensions.add(dimension)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate dimension [%s] found in dimensions_order for composite index field [%s]", + dimension, + field + ) + ); + } + dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension))); + } + uniqueDimensions = null; + Set uniqueMetricFields = new HashSet<>(); + for (String metricField : metricFields) { + if (!uniqueMetricFields.add(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric field [%s] found in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + Settings metricSettings = metricsConfig.get(metricField); + if (metricSettings == null) { + // fill cluster level defaults in create flow as part of CompositeIndexSupplier + metrics.add(new Metric(metricField, new ArrayList<>())); + } else { + String name = metricSettings.get(FIELD, metricField); + List metricsList = metricSettings.getAsList(METRICS); + if (metricsList.isEmpty()) { + // fill cluster level defaults in create flow as part of CompositeIndexSupplier + metrics.add(new Metric(name, new ArrayList<>())); + } else { + List metricTypes = new ArrayList<>(); + Set uniqueMetricTypes = new HashSet<>(); + for (String metric : metricsList) { + if (!uniqueMetricTypes.add(metric)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric type [%s] found in metrics for composite index field [%s]", + metric, + field + ) + ); + } + metricTypes.add(MetricType.fromTypeName(metric)); + } + uniqueMetricTypes = null; + metrics.add(new Metric(name, metricTypes)); + } + } + } + uniqueMetricFields = null; + + IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); + Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); + CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder); + return new CompositeField(field, dimensions, metrics, compositeFieldSpec); + } + + public static Rounding.DateTimeUnit getTimeUnit(String expression) { + if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); + } + return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + } + + /** + * Dimension factory based on field type + */ + private static class DimensionFactory { + static Dimension create(String dimension, Settings settings) { + if (settings == null) { + return new Dimension(dimension); + } + String field = settings.get(FIELD, dimension); + String type = settings.get(TYPE, DimensionType.DEFAULT.getTypeName()); + switch (DimensionType.fromTypeName(type)) { + case DEFAULT: + return new Dimension(field); + case DATE: + return new DateDimension(field, settings); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension type [%s] in composite index config", type) + ); + } + } + + static Dimension createEmptyMappedDimension(Dimension dimension, MappedFieldType type) { + if (type instanceof DateFieldMapper.DateFieldType) { + return new DateDimension(dimension.getField(), new ArrayList<>()); + } + return dimension; + } + } + + /** + * The type of dimension source fields + * Default fields are of Numeric type + */ + private enum DimensionType { + DEFAULT("default"), + DATE("date"); + + private final String typeName; + + DimensionType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static DimensionType fromTypeName(String typeName) { + for (DimensionType dimensionType : DimensionType.values()) { + if (dimensionType.getTypeName().equalsIgnoreCase(typeName)) { + return dimensionType; + } + } + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension type in composite index config: [%s] ", typeName) + ); + } + } + + /** + * Composite field spec factory based on index mode + */ + private static class CompositeFieldSpecFactory { + static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + if (settings == null) { + return new StarTreeFieldSpec(10000, new ArrayList<>(), StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP); + } + switch (indexMode) { + case STARTREE: + return buildStarTreeFieldSpec(settings, dimensions); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid index mode [%s] in composite index config", indexMode) + ); + } + } + } + + private static StarTreeFieldSpec buildStarTreeFieldSpec(Settings settings, List dimensions) { + StarTreeFieldSpec.StarTreeBuildMode buildMode = StarTreeFieldSpec.StarTreeBuildMode.fromTypeName( + settings.get(STAR_TREE_BUILD_MODE, StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP.getTypeName()) + ); + // Fill default value as part of create flow as part of supplier + int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, Integer.MAX_VALUE); + if (maxLeafDocs < 1) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid max_leaf_docs [%s] in composite index config", maxLeafDocs) + ); + } + List skipStarNodeCreationInDims = settings.getAsList(SKIP_STAR_NODE_CREATION_FOR_DIMS, new ArrayList<>()); + for (String dim : skipStarNodeCreationInDims) { + if (!dimensions.contains(dim)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension [%s] in skip_star_node_creation_for_dims", dim) + ); + } + } + return new StarTreeFieldSpec(maxLeafDocs, skipStarNodeCreationInDims, buildMode); + } + + /** + * Enum for index mode of the underlying composite index + * The default and only index supported right now is star tree index + */ + private enum IndexMode { + STARTREE("startree"); + + private final String typeName; + + IndexMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static IndexMode fromTypeName(String typeName) { + for (IndexMode indexType : IndexMode.values()) { + if (indexType.getTypeName().equalsIgnoreCase(typeName)) { + return indexType; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid index mode in composite index config: [%s] ", typeName)); + } + } + + /** + * Returns the composite fields built based on compositeIndexConfig index settings + */ + public List getCompositeFields() { + return compositeFields; + } + + /** + * Returns whether there are any composite fields as part of the compositeIndexConfig + */ + public boolean hasCompositeFields() { + return !compositeFields.isEmpty(); + } + + /** + * Validates the composite fields based on IndexSettingDefaults and the mappedFieldType + * Updates CompositeIndexConfig with newer, completely updated composite fields + * + */ + public CompositeIndexConfig validateAndGetCompositeIndexConfig( + Function fieldTypeLookup, + CompositeIndexSettings compositeIndexSettings + ) { + if (hasCompositeFields() == false) { + return null; + } + if (!compositeIndexSettings.isEnabled()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index cannot be created, enable it using [%s] setting", + COMPOSITE_INDEX_ENABLED_SETTING.getKey() + ) + ); + } + if (compositeFields.size() > compositeIndexSettings.getMaxFields()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] fields", compositeIndexSettings.getMaxFields()) + ); + } + List validatedAndMappedCompositeFields = new ArrayList<>(); + for (CompositeField compositeField : compositeFields) { + if (compositeField.getDimensionsOrder().size() > compositeIndexSettings.getMaxDimensions()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", compositeIndexSettings.getMaxDimensions()) + ); + } + List dimensions = new ArrayList<>(); + for (Dimension dimension : compositeField.getDimensionsOrder()) { + validateCompositeDimensionField(dimension.getField(), fieldTypeLookup, compositeField.getName()); + dimension = mapDimension(dimension, fieldTypeLookup.apply(dimension.getField())); + dimension.setDefaults(compositeIndexSettings); + dimensions.add(dimension); + } + List metrics = new ArrayList<>(); + for (Metric metric : compositeField.getMetrics()) { + validateCompositeMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); + metric.setDefaults(compositeIndexSettings); + metrics.add(metric); + } + compositeField.getSpec().setDefaults(compositeIndexSettings); + validatedAndMappedCompositeFields.add( + new CompositeField(compositeField.getName(), dimensions, metrics, compositeField.getSpec()) + ); + } + this.compositeFields.clear(); + this.compositeFields.addAll(validatedAndMappedCompositeFields); + return this; + } + + /** + * Maps the dimension to right dimension type based on MappedFieldType + */ + private Dimension mapDimension(Dimension dimension, MappedFieldType fieldType) { + if (!isDimensionMappedToFieldType(dimension, fieldType)) { + return DimensionFactory.createEmptyMappedDimension(dimension, fieldType); + } + return dimension; + } + + /** + * Checks whether dimension field type is same as the source field type + */ + private boolean isDimensionMappedToFieldType(Dimension dimension, MappedFieldType fieldType) { + if (fieldType instanceof DateFieldMapper.DateFieldType) { + return dimension instanceof DateDimension; + } + return true; + } + + /** + * Validations : + * The dimension field should be one of the source fields of the index + * The dimension fields must be aggregation compatible (doc values + field data supported) + * The dimension fields should be of numberField type / dateField type + * + */ + private void validateCompositeDimensionField( + String field, + Function fieldTypeLookup, + String compositeFieldName + ) { + final MappedFieldType ft = fieldTypeLookup.apply(field); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown dimension field [%s] as part of composite field [%s]", field, compositeFieldName) + ); + } + if (!isAllowedDimensionFieldType(ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index is not supported for the dimension field [%s] with field type [%s] as part of " + + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + // doc values not present / field data not supported + if (!ft.isAggregatable()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + } + + /** + * Validations : + * The metric field should be one of the source fields of the index + * The metric fields must be aggregation compatible (doc values + field data supported) + * The metric fields should be of numberField type + * + */ + private void validateCompositeMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { + final MappedFieldType ft = fieldTypeLookup.apply(field); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown metric field [%s] as part of composite field [%s]", field, compositeFieldName) + ); + } + if (!isAllowedMetricFieldType(ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index is not supported for the metric field [%s] with field type [%s] as part of " + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + if (!ft.isAggregatable()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the composite index metric field [%s] with field type [%s] as part of " + + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + } + + private static boolean isAllowedDimensionFieldType(MappedFieldType fieldType) { + return ALLOWED_DIMENSION_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); + } + + private static boolean isAllowedMetricFieldType(MappedFieldType fieldType) { + return ALLOWED_METRIC_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java new file mode 100644 index 0000000000000..59bdfda5469ea --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java @@ -0,0 +1,179 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.util.FeatureFlags; + +import java.util.Arrays; +import java.util.List; + +/** + * Cluster level settings which configures defaults for composite index + */ +@ExperimentalApi +public class CompositeIndexSettings { + /** + * This cluster level setting determines whether composite index is enabled or not + */ + public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( + "indices.composite_index.enabled", + false, + value -> { + if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting determines the max number of composite fields that can be part of composite index config. For each + * composite field, we will generate associated composite index. (eg : star tree index per field ) + */ + public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( + "indices.composite_index.max_fields", + 1, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting determines the max number of dimensions that can be part of composite index field. Number of + * dimensions and associated cardinality has direct effect of composite index size and query performance. + */ + public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( + "indices.composite_index.field.max_dimensions", + 10, + 2, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and + * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa + *

+ * We can remove this later or change it to an enum based constant setting. + * + * @opensearch.experimental + */ + public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( + "indices.composite_index.startree.default.max_leaf_docs", + 10000, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Default intervals for date dimension as part of composite fields + */ + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + "indices.composite_index.field.default.date_intervals", + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + CompositeIndexConfig::getTimeUnit, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( + "indices.composite_index.field.default.metrics", + Arrays.asList( + MetricType.AVG.toString(), + MetricType.COUNT.toString(), + MetricType.SUM.toString(), + MetricType.MAX.toString(), + MetricType.MIN.toString() + ), + MetricType::fromTypeName, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private volatile int maxLeafDocs; + + private volatile List defaultDateIntervals; + + private volatile List defaultMetrics; + private volatile int maxDimensions; + private volatile int maxFields; + private volatile boolean enabled; + + public CompositeIndexSettings(ClusterSettings clusterSettings) { + this.setMaxLeafDocs(clusterSettings.get(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); + this.setDefaultDateIntervals(clusterSettings.get(DEFAULT_DATE_INTERVALS)); + this.setDefaultMetrics(clusterSettings.get(DEFAULT_METRICS_LIST)); + this.setMaxDimensions(clusterSettings.get(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); + this.setMaxFields(clusterSettings.get(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); + this.setEnabled(clusterSettings.get(COMPOSITE_INDEX_ENABLED_SETTING)); + + clusterSettings.addSettingsUpdateConsumer(STAR_TREE_DEFAULT_MAX_LEAF_DOCS, this::setMaxLeafDocs); + clusterSettings.addSettingsUpdateConsumer(DEFAULT_DATE_INTERVALS, this::setDefaultDateIntervals); + clusterSettings.addSettingsUpdateConsumer(DEFAULT_METRICS_LIST, this::setDefaultMetrics); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, this::setMaxDimensions); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_FIELDS_SETTING, this::setMaxFields); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setEnabled); + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public void setMaxLeafDocs(int maxLeafDocs) { + this.maxLeafDocs = maxLeafDocs; + } + + public void setDefaultDateIntervals(List defaultDateIntervals) { + this.defaultDateIntervals = defaultDateIntervals; + } + + public void setDefaultMetrics(List defaultMetrics) { + this.defaultMetrics = defaultMetrics; + } + + public void setMaxDimensions(int maxDimensions) { + this.maxDimensions = maxDimensions; + } + + public void setMaxFields(int maxFields) { + this.maxFields = maxFields; + } + + public int getMaxDimensions() { + return maxDimensions; + } + + public int getMaxFields() { + return maxFields; + } + + public int getMaxLeafDocs() { + return maxLeafDocs; + } + + public boolean isEnabled() { + return enabled; + } + + public List getDefaultDateIntervals() { + return defaultDateIntervals; + } + + public List getDefaultMetrics() { + return defaultMetrics; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java new file mode 100644 index 0000000000000..40145f9f80ef3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; + +/** + * Date dimension class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class DateDimension extends Dimension { + private final List calendarIntervals; + + public DateDimension(String name, Settings settings) { + super(name); + List intervalStrings = settings.getAsList("calendar_interval"); + if (intervalStrings == null || intervalStrings.isEmpty()) { + this.calendarIntervals = new ArrayList<>(); + } else { + this.calendarIntervals = new ArrayList<>(); + for (String interval : intervalStrings) { + this.calendarIntervals.add(CompositeIndexConfig.getTimeUnit(interval)); + } + } + } + + public DateDimension(String name, List calendarIntervals) { + super(name); + this.calendarIntervals = calendarIntervals; + } + + @Override + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (calendarIntervals.isEmpty()) { + this.calendarIntervals.addAll(compositeIndexSettings.getDefaultDateIntervals()); + } + } + + public List getIntervals() { + return calendarIntervals; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java new file mode 100644 index 0000000000000..c06df856d5b8a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Composite index dimension base class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Dimension { + private final String field; + + public Dimension(String field) { + this.field = field; + } + + public String getField() { + return field; + } + + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + // no implementation + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java new file mode 100644 index 0000000000000..dddc3c795078c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; + +/** + * Holds details of metrics field as part of composite field + */ +@ExperimentalApi +public class Metric { + private final String field; + private final List metrics; + + public Metric(String field, List metrics) { + this.field = field; + this.metrics = metrics; + } + + public String getField() { + return field; + } + + public List getMetrics() { + return metrics; + } + + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (metrics.isEmpty()) { + metrics.addAll(compositeIndexSettings.getDefaultMetrics()); + } + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java new file mode 100644 index 0000000000000..9261279fa5957 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Supported metric types for composite index + */ +@ExperimentalApi +public enum MetricType { + COUNT("count"), + AVG("avg"), + SUM("sum"), + MIN("min"), + MAX("max"); + + private final String typeName; + + MetricType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static MetricType fromTypeName(String typeName) { + for (MetricType metric : MetricType.values()) { + if (metric.getTypeName().equalsIgnoreCase(typeName)) { + return metric; + } + } + throw new IllegalArgumentException("Invalid metric type: " + typeName); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java new file mode 100644 index 0000000000000..c38f8ca8813c7 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Star tree index specific settings for a composite field. + */ +@ExperimentalApi +public class StarTreeFieldSpec implements CompositeFieldSpec { + + private final AtomicInteger maxLeafDocs = new AtomicInteger(); + private final List skipStarNodeCreationInDims; + private final StarTreeBuildMode buildMode; + + public StarTreeFieldSpec(int maxLeafDocs, List skipStarNodeCreationInDims, StarTreeBuildMode buildMode) { + this.maxLeafDocs.set(maxLeafDocs); + this.skipStarNodeCreationInDims = skipStarNodeCreationInDims; + this.buildMode = buildMode; + } + + /** + * Star tree build mode using which sorting and aggregations are performed during index creation. + * + * @opensearch.experimental + */ + @ExperimentalApi + public enum StarTreeBuildMode { + ON_HEAP("onheap"), + OFF_HEAP("offheap"); + + private final String typeName; + + StarTreeBuildMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static StarTreeBuildMode fromTypeName(String typeName) { + for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) { + if (starTreeBuildMode.getTypeName().equalsIgnoreCase(typeName)) { + return starTreeBuildMode; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", typeName)); + } + } + + public int maxLeafDocs() { + return maxLeafDocs.get(); + } + + @Override + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (maxLeafDocs.get() == Integer.MAX_VALUE) { + maxLeafDocs.set(compositeIndexSettings.getMaxLeafDocs()); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java new file mode 100644 index 0000000000000..59f18efec26b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Core classes for handling composite indices. + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex; diff --git a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java new file mode 100644 index 0000000000000..be49269f0264c --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.compositeindex.CompositeIndexSettings; + +/** + * Utility to provide a {@link CompositeIndexSettings} instance containing all defaults + * + * @opensearch.experimental + */ +@ExperimentalApi +public final class DefaultCompositeIndexSettings { + private DefaultCompositeIndexSettings() {} + + public static final CompositeIndexSettings INSTANCE = new CompositeIndexSettings( + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 251be8a990055..b1fb2f77e2981 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -106,6 +106,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.cache.request.ShardRequestCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineConfigFactory; @@ -354,6 +355,7 @@ public class IndicesService extends AbstractLifecycleComponent private final BiFunction translogFactorySupplier; private volatile TimeValue clusterDefaultRefreshInterval; private final SearchRequestStats searchRequestStats; + private final CompositeIndexSettings compositeIndexSettings; @Override protected void doStart() { @@ -388,7 +390,8 @@ public IndicesService( @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, RecoverySettings recoverySettings, CacheService cacheService, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) { this.settings = settings; this.threadPool = threadPool; @@ -495,6 +498,7 @@ protected void closeInternal() { .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate); this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; + this.compositeIndexSettings = compositeIndexSettings; } /** @@ -903,7 +907,8 @@ private synchronized IndexService createIndexService( translogFactorySupplier, this::getClusterDefaultRefreshInterval, this.recoverySettings, - this.remoteStoreSettings + this.remoteStoreSettings, + this.compositeIndexSettings ); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 9462aeddbd0e4..949cc2982ad6f 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -146,6 +146,7 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationStatsTracker; import org.opensearch.index.analysis.AnalysisRegistry; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; @@ -833,6 +834,8 @@ protected Node( final SearchRequestStats searchRequestStats = new SearchRequestStats(clusterService.getClusterSettings()); final SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService); + final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterService.getClusterSettings()); + remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); CacheModule cacheModule = new CacheModule(pluginsService.filterPlugins(CachePlugin.class), settings); CacheService cacheService = cacheModule.getCacheService(); @@ -863,7 +866,8 @@ protected Node( remoteStoreStatsTrackerFactory, recoverySettings, cacheService, - remoteStoreSettings + remoteStoreSettings, + compositeIndexSettings ); final IngestService ingestService = new IngestService( diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 3d6a54055d3d5..d5269252681a7 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -1118,6 +1118,7 @@ private IndicesService mockIndicesServices(DocumentMapper documentMapper) throws when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() { }); when(indexService.getIndexSortSupplier()).thenReturn(() -> null); + when(indexService.getCompositeIndexConfigSupplier()).thenReturn(() -> null); // noinspection unchecked return ((CheckedFunction) invocationOnMock.getArguments()[1]).apply(indexService); }); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 4ce4936c047d9..8f45a872e752c 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -99,6 +99,7 @@ import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.TranslogFactory; +import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesQueryCache; @@ -264,7 +265,8 @@ private IndexService newIndexService(IndexModule module) throws IOException { translogFactorySupplier, () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + DefaultCompositeIndexSettings.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java new file mode 100644 index 0000000000000..98a017bf1acbd --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java @@ -0,0 +1,233 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.opensearch.index.IndexSettingsTests.newIndexMeta; + +/** + * Composite index config settings unit tests + */ +public class CompositeIndexConfigSettingsTests extends OpenSearchTestCase { + private static IndexSettings indexSettings(Settings settings) { + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); + } + + public void testDefaultSettings() { + Settings settings = Settings.EMPTY; + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertFalse(compositeIndexConfig.hasCompositeFields()); + } + + public void testMinimumMetrics() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("metrics is required for composite index field [my_field]", exception.getMessage()); + } + + public void testMinimumDimensions() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Atleast two dimensions are required to build composite index field [my_field]", exception.getMessage()); + } + + public void testInvalidDimensionType() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Invalid dimension type in composite index config: [invalid] ", exception.getMessage()); + } + + public void testInvalidIndexMode() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .put("index.composite_index.config.my_field.index_mode", "invalid") + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Invalid index mode in composite index config: [invalid] ", exception.getMessage()); + } + + public void testValidCompositeIndexConfig() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + } + + public void testCompositeIndexMultipleFields() { + Settings settings = Settings.builder() + .put("indices.composite_index.max_fields", 2) + .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.field1.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.field2.dimensions_order", Arrays.asList("dim3", "dim4")) + .put("index.composite_index.config.field2.dimensions_config.dim3.field", "dim3_field") + .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") + .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(2, compositeIndexConfig.getCompositeFields().size()); + } + + public void testCompositeIndexDateIntervalsSetting() { + Settings settings = Settings.builder() + .putList("indices.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "date") + .putList("index.composite_index.config.my_field.dimensions_config.dim1.calendar_interval", Arrays.asList("day", "week")) + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + List expectedIntervals = Arrays.asList( + Rounding.DateTimeUnit.DAY_OF_MONTH, + Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR + ); + assertEquals(expectedIntervals, ((DateDimension) compositeField.getDimensionsOrder().get(0)).getIntervals()); + } + + public void testCompositeIndexMetricsSetting() { + Settings settings = Settings.builder() + .putList("indices.composite_index.field.default.metrics", Arrays.asList("count", "max")) + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.my_field.metrics_config.metric1.metrics", Arrays.asList("count", "max")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + List expectedMetrics = Arrays.asList(MetricType.COUNT, MetricType.MAX); + assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + } + + public void testValidateWithoutCompositeSettingEnabled() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "default") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.LONG)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric2", NumberFieldMapper.NumberType.LONG)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterSettings); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, compositeIndexSettings) + ); + assertEquals( + "composite index cannot be created, enable it using [indices.composite_index.enabled] setting", + exception.getMessage() + ); + + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + } + + public void testEnabledWithFFOff() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(testWithEnabledSettings(settings)); + Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); + ClusterSettings clusterSettings = new ClusterSettings(settings1, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new CompositeIndexSettings(clusterSettings) + ); + assertEquals( + "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + exception.getMessage() + ); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + assertTrue(compositeField.getDimensionsOrder().get(0) instanceof Dimension); + assertTrue(compositeField.getDimensionsOrder().get(1) instanceof DateDimension); + } + + private IndexSettings createIndexSettings(Settings settings) { + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); + } + + public IndexSettings testWithEnabledSettings(Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings1), Settings.EMPTY); + return indexSettings; + } + +} diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 8f1d58cf201e9..a59e061330b91 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -218,6 +218,7 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() { }); when(indexService.getIndexSortSupplier()).thenReturn(() -> null); + when(indexService.getCompositeIndexConfigSupplier()).thenReturn(() -> null); // noinspection unchecked return ((CheckedFunction) invocationOnMock.getArguments()[1]).apply(indexService); }); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 460aaa08a224d..1b44c2c3008d9 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -193,6 +193,7 @@ import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheStats; +import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesService; @@ -2086,7 +2087,8 @@ public void onFailure(final Exception e) { new RemoteStoreStatsTrackerFactory(clusterService, settings), DefaultRecoverySettings.INSTANCE, new CacheModule(new ArrayList<>(), settings).getCacheService(), - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + DefaultCompositeIndexSettings.INSTANCE ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService(