From 8808c6e525b73c94a785c16c61d2f4e01aaa313f Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Fri, 31 May 2024 23:05:25 +0530 Subject: [PATCH 1/2] Star tree index config changes Signed-off-by: Bharathwaj G --- CHANGELOG.md | 1 + distribution/src/config/opensearch.yml | 4 + .../metadata/MetadataCreateIndexService.java | 2 + .../common/settings/ClusterSettings.java | 11 +- .../common/settings/FeatureFlagSettings.java | 3 +- .../common/settings/IndexScopedSettings.java | 4 +- .../opensearch/common/util/FeatureFlags.java | 11 +- .../org/opensearch/index/IndexModule.java | 7 +- .../org/opensearch/index/IndexService.java | 19 +- .../org/opensearch/index/IndexSettings.java | 8 + .../index/compositeindex/CompositeField.java | 49 ++ .../compositeindex/CompositeFieldSpec.java | 22 + .../compositeindex/CompositeIndexConfig.java | 505 ++++++++++++++++++ .../CompositeIndexSettings.java | 179 +++++++ .../index/compositeindex/DateDimension.java | 55 ++ .../index/compositeindex/Dimension.java | 33 ++ .../index/compositeindex/Metric.java | 42 ++ .../index/compositeindex/MetricType.java | 42 ++ .../compositeindex/StarTreeFieldSpec.java | 73 +++ .../index/compositeindex/package-info.java | 13 + .../DefaultCompositeIndexSettings.java | 28 + .../opensearch/indices/IndicesService.java | 9 +- .../main/java/org/opensearch/node/Node.java | 6 +- .../MetadataRolloverServiceTests.java | 1 + .../opensearch/index/IndexModuleTests.java | 4 +- .../CompositeIndexConfigSettingsTests.java | 233 ++++++++ .../indices/cluster/ClusterStateChanges.java | 1 + .../snapshots/SnapshotResiliencyTests.java | 4 +- 28 files changed, 1356 insertions(+), 13 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/Dimension.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/Metric.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/MetricType.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/package-info.java create mode 100644 server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b5dd289ec85..df4c2c3bd5480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Store] Add dynamic cluster settings to set timeout for segments upload to Remote Store ([#13679](https://github.com/opensearch-project/OpenSearch/pull/13679)) - [Remote Store] Upload translog checkpoint as object metadata to translog.tlog([#13637](https://github.com/opensearch-project/OpenSearch/pull/13637)) - Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819)) +- [Star Tree Index] Star tree index config changes ([#13917](https://github.com/opensearch-project/OpenSearch/pull/13917)) ### Dependencies - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559)) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 10bab9b3fce92..b44248cf74335 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -125,3 +125,7 @@ ${path.logs} # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting. # #opensearch.experimental.feature.pluggable.caching.enabled: false +# +# Gates the functionality of star tree index, which improves the performance of search aggregations. +# +#opensearch.experimental.feature.composite_index.enabled: true diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 16edec112f123..59e78605c33cb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1324,6 +1324,8 @@ private static void updateIndexMappingsAndBuildSortOrder( // at this point. The validation will take place later in the process // (when all shards are copied in a single place). indexService.getIndexSortSupplier().get(); + // validate composite index fields + indexService.getCompositeIndexConfigSupplier().get(); } if (request.dataStreamName() != null) { MetadataCreateDataStreamService.validateTimestampFieldMapping(mapperService); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index a2be2ea4510e0..7db247119a282 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -113,6 +113,7 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.remote.RemoteStorePressureSettings; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.store.remote.filecache.FileCache; @@ -741,7 +742,15 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS, - RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA + RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, + + // Composite index settings + CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING, + CompositeIndexSettings.COMPOSITE_INDEX_MAX_FIELDS_SETTING, + CompositeIndexSettings.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, + CompositeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, + CompositeIndexSettings.DEFAULT_METRICS_LIST, + CompositeIndexSettings.DEFAULT_DATE_INTERVALS ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 7a364de1c5dc6..2f21d508673c8 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -36,6 +36,7 @@ protected FeatureFlagSettings( FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, - FeatureFlags.PLUGGABLE_CACHE_SETTING + FeatureFlags.PLUGGABLE_CACHE_SETTING, + FeatureFlags.COMPOSITE_INDEX_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 980c432774f6e..9b4521cc3647a 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -249,8 +249,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { } } }, Property.IndexScope), // this allows similarity settings to be passed - Setting.groupSetting("index.analysis.", Property.IndexScope) // this allows analysis settings to be passed - + Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed + Setting.groupSetting("index.composite_index.", Property.IndexScope) // this allows composite index settings to be passed ) ) ); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 62cfbd861d4d9..fe8081d888e98 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -67,6 +67,12 @@ public class FeatureFlags { */ public static final String PLUGGABLE_CACHE = "opensearch.experimental.feature.pluggable.caching.enabled"; + /** + * Gates the functionality of composite index i.e. star tree index, which improves the performance of search + * aggregations. + */ + public static final String COMPOSITE_INDEX = "opensearch.experimental.feature.composite_index.enabled"; + public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_STORE_MIGRATION_EXPERIMENTAL, false, @@ -89,6 +95,8 @@ public class FeatureFlags { public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); + public static final Setting COMPOSITE_INDEX_SETTING = Setting.boolSetting(COMPOSITE_INDEX, false, Property.NodeScope); + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -96,7 +104,8 @@ public class FeatureFlags { TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, - PLUGGABLE_CACHE_SETTING + PLUGGABLE_CACHE_SETTING, + COMPOSITE_INDEX_SETTING ); /** * Should store the settings from opensearch.yml. diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 3c4cb4fd596c1..aaec6bfec2123 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -66,6 +66,7 @@ import org.opensearch.index.cache.query.DisabledQueryCache; import org.opensearch.index.cache.query.IndexQueryCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -606,7 +607,8 @@ public IndexService newIndexService( BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -665,7 +667,8 @@ public IndexService newIndexService( translogFactorySupplier, clusterDefaultRefreshIntervalSupplier, recoverySettings, - remoteStoreSettings + remoteStoreSettings, + compositeIndexSettings ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index e501d7eff3f81..6f994e8f67c5b 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -72,6 +72,8 @@ import org.opensearch.index.cache.IndexCache; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexConfig; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -183,6 +185,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final CircuitBreakerService circuitBreakerService; private final IndexNameExpressionResolver expressionResolver; private final Supplier indexSortSupplier; + private final Supplier compositeIndexConfigSupplier; private final ValuesSourceRegistry valuesSourceRegistry; private final BiFunction translogFactorySupplier; private final Supplier clusterDefaultRefreshIntervalSupplier; @@ -223,7 +226,8 @@ public IndexService( BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -261,6 +265,14 @@ public IndexService( } else { this.indexSortSupplier = () -> null; } + + if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) { + this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig() + .validateAndGetCompositeIndexConfig(mapperService::fieldType, compositeIndexSettings); + } else { + this.compositeIndexConfigSupplier = () -> null; + } + indexFieldData.setListener(new FieldDataCacheListener(this)); this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this)); this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool)); @@ -273,6 +285,7 @@ public IndexService( this.bitsetFilterCache = null; this.warmer = null; this.indexCache = null; + this.compositeIndexConfigSupplier = () -> null; } this.shardStoreDeleter = shardStoreDeleter; @@ -385,6 +398,10 @@ public Supplier getIndexSortSupplier() { return indexSortSupplier; } + public Supplier getCompositeIndexConfigSupplier() { + return compositeIndexConfigSupplier; + } + public synchronized void close(final String reason, boolean delete) throws IOException { if (closed.compareAndSet(false, true)) { deleted.compareAndSet(false, delete); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 2b0a62c18d5a7..4cfdffdce4ee0 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -48,6 +48,7 @@ import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.Index; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.Translog; @@ -753,6 +754,8 @@ public static IndexMergePolicy fromString(String text) { private final LogByteSizeMergePolicyProvider logByteSizeMergePolicyProvider; private final IndexSortConfig indexSortConfig; private final IndexScopedSettings scopedSettings; + + private final CompositeIndexConfig compositeIndexConfig; private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); private final boolean softDeleteEnabled; private volatile long softDeleteRetentionOperations; @@ -966,6 +969,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.tieredMergePolicyProvider = new TieredMergePolicyProvider(logger, this); this.logByteSizeMergePolicyProvider = new LogByteSizeMergePolicyProvider(logger, this); this.indexSortConfig = new IndexSortConfig(this); + this.compositeIndexConfig = new CompositeIndexConfig(this); searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER); defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE); setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING)); @@ -1720,6 +1724,10 @@ public IndexSortConfig getIndexSortConfig() { return indexSortConfig; } + public CompositeIndexConfig getCompositeIndexConfig() { + return compositeIndexConfig; + } + public IndexScopedSettings getScopedSettings() { return scopedSettings; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java new file mode 100644 index 0000000000000..a0b95b0409da0 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; + +/** + * Composite field which contains dimensions, metrics and index mode specific specs + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeField { + private final String name; + private final List dimensionsOrder; + private final List metrics; + private final CompositeFieldSpec compositeFieldSpec; + + public CompositeField(String name, List dimensions, List metrics, CompositeFieldSpec compositeFieldSpec) { + this.name = name; + this.dimensionsOrder = dimensions; + this.metrics = metrics; + this.compositeFieldSpec = compositeFieldSpec; + } + + public String getName() { + return name; + } + + public List getDimensionsOrder() { + return dimensionsOrder; + } + + public List getMetrics() { + return metrics; + } + + public CompositeFieldSpec getSpec() { + return compositeFieldSpec; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java new file mode 100644 index 0000000000000..51a2b454abcd9 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * CompositeFieldSpec interface. + * + * @opensearch.experimental + */ + +@ExperimentalApi +public interface CompositeFieldSpec { + void setDefaults(CompositeIndexSettings compositeIndexSettings); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java new file mode 100644 index 0000000000000..308c4b62d3996 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java @@ -0,0 +1,505 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static org.opensearch.index.compositeindex.CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING; + +/** + * Configuration of composite index containing list of composite fields. + * Each composite field contains dimensions, metrics along with composite index (eg: star tree) specific settings. + * Each composite field will generate a composite index in indexing flow. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeIndexConfig { + private static final Set> ALLOWED_DIMENSION_MAPPED_FIELD_TYPES = Set.of( + NumberFieldMapper.NumberFieldType.class, + DateFieldMapper.DateFieldType.class + ); + + private static final Set> ALLOWED_METRIC_MAPPED_FIELD_TYPES = Set.of( + NumberFieldMapper.NumberFieldType.class + ); + + private static final String COMPOSITE_INDEX_CONFIG = "index.composite_index.config"; + private static final String DIMENSIONS_ORDER = "dimensions_order"; + private static final String DIMENSIONS_CONFIG = "dimensions_config"; + private static final String METRICS = "metrics"; + private static final String METRICS_CONFIG = "metrics_config"; + private static final String FIELD = "field"; + private static final String TYPE = "type"; + private static final String INDEX_MODE = "index_mode"; + private static final String STAR_TREE_BUILD_MODE = "build_mode"; + private static final String MAX_LEAF_DOCS = "max_leaf_docs"; + private static final String SKIP_STAR_NODE_CREATION_FOR_DIMS = "skip_star_node_creation_for_dimensions"; + private static final String SPEC = "_spec"; + private final List compositeFields = new ArrayList<>(); + + public CompositeIndexConfig(IndexSettings indexSettings) { + + final Map compositeIndexSettings = indexSettings.getSettings().getGroups(COMPOSITE_INDEX_CONFIG); + Set fields = compositeIndexSettings.keySet(); + for (String field : fields) { + compositeFields.add(buildCompositeField(field, compositeIndexSettings.get(field))); + } + } + + /** + * This returns composite field after performing basic validations and doesn't do field type validations etc + * + */ + private CompositeField buildCompositeField(String field, Settings compositeFieldSettings) { + List dimensions = new ArrayList<>(); + List metrics = new ArrayList<>(); + List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); + if (dimensionsOrder == null || dimensionsOrder.isEmpty()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "dimensions_order is required for composite index field [%s]", field) + ); + } + if (dimensionsOrder.size() < 2) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Atleast two dimensions are required to build composite index field [%s]", field) + ); + } + + Map dimConfig = compositeFieldSettings.getGroups(DIMENSIONS_CONFIG); + + for (String dimension : dimConfig.keySet()) { + if (!dimensionsOrder.contains(dimension)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "dimension [%s] is not present in dimensions_order for composite index field [%s]", + dimension, + field + ) + ); + } + } + + List metricFields = compositeFieldSettings.getAsList(METRICS); + if (metricFields == null || metricFields.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "metrics is required for composite index field [%s]", field)); + } + Map metricsConfig = compositeFieldSettings.getGroups(METRICS_CONFIG); + + for (String metricField : metricsConfig.keySet()) { + if (!metricFields.contains(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "metric field [%s] is not present in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + } + + Set uniqueDimensions = new HashSet<>(); + for (String dimension : dimensionsOrder) { + if (!uniqueDimensions.add(dimension)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate dimension [%s] found in dimensions_order for composite index field [%s]", + dimension, + field + ) + ); + } + dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension))); + } + uniqueDimensions = null; + Set uniqueMetricFields = new HashSet<>(); + for (String metricField : metricFields) { + if (!uniqueMetricFields.add(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric field [%s] found in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + Settings metricSettings = metricsConfig.get(metricField); + if (metricSettings == null) { + // fill cluster level defaults in create flow as part of CompositeIndexSupplier + metrics.add(new Metric(metricField, new ArrayList<>())); + } else { + String name = metricSettings.get(FIELD, metricField); + List metricsList = metricSettings.getAsList(METRICS); + if (metricsList.isEmpty()) { + // fill cluster level defaults in create flow as part of CompositeIndexSupplier + metrics.add(new Metric(name, new ArrayList<>())); + } else { + List metricTypes = new ArrayList<>(); + Set uniqueMetricTypes = new HashSet<>(); + for (String metric : metricsList) { + if (!uniqueMetricTypes.add(metric)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric type [%s] found in metrics for composite index field [%s]", + metric, + field + ) + ); + } + metricTypes.add(MetricType.fromTypeName(metric)); + } + uniqueMetricTypes = null; + metrics.add(new Metric(name, metricTypes)); + } + } + } + uniqueMetricFields = null; + + IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); + Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); + CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder); + return new CompositeField(field, dimensions, metrics, compositeFieldSpec); + } + + public static Rounding.DateTimeUnit getTimeUnit(String expression) { + if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); + } + return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + } + + /** + * Dimension factory based on field type + */ + private static class DimensionFactory { + static Dimension create(String dimension, Settings settings) { + if (settings == null) { + return new Dimension(dimension); + } + String field = settings.get(FIELD, dimension); + String type = settings.get(TYPE, DimensionType.DEFAULT.getTypeName()); + switch (DimensionType.fromTypeName(type)) { + case DEFAULT: + return new Dimension(field); + case DATE: + return new DateDimension(field, settings); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension type [%s] in composite index config", type) + ); + } + } + + static Dimension createEmptyMappedDimension(Dimension dimension, MappedFieldType type) { + if (type instanceof DateFieldMapper.DateFieldType) { + return new DateDimension(dimension.getField(), new ArrayList<>()); + } + return dimension; + } + } + + /** + * The type of dimension source fields + * Default fields are of Numeric type + */ + private enum DimensionType { + DEFAULT("default"), + DATE("date"); + + private final String typeName; + + DimensionType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static DimensionType fromTypeName(String typeName) { + for (DimensionType dimensionType : DimensionType.values()) { + if (dimensionType.getTypeName().equalsIgnoreCase(typeName)) { + return dimensionType; + } + } + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension type in composite index config: [%s] ", typeName) + ); + } + } + + /** + * Composite field spec factory based on index mode + */ + private static class CompositeFieldSpecFactory { + static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + if (settings == null) { + return new StarTreeFieldSpec(10000, new ArrayList<>(), StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP); + } + switch (indexMode) { + case STARTREE: + return buildStarTreeFieldSpec(settings, dimensions); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid index mode [%s] in composite index config", indexMode) + ); + } + } + } + + private static StarTreeFieldSpec buildStarTreeFieldSpec(Settings settings, List dimensions) { + StarTreeFieldSpec.StarTreeBuildMode buildMode = StarTreeFieldSpec.StarTreeBuildMode.fromTypeName( + settings.get(STAR_TREE_BUILD_MODE, StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP.getTypeName()) + ); + // Fill default value as part of create flow as part of supplier + int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, Integer.MAX_VALUE); + if (maxLeafDocs < 1) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid max_leaf_docs [%s] in composite index config", maxLeafDocs) + ); + } + List skipStarNodeCreationInDims = settings.getAsList(SKIP_STAR_NODE_CREATION_FOR_DIMS, new ArrayList<>()); + for (String dim : skipStarNodeCreationInDims) { + if (!dimensions.contains(dim)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension [%s] in skip_star_node_creation_for_dims", dim) + ); + } + } + return new StarTreeFieldSpec(maxLeafDocs, skipStarNodeCreationInDims, buildMode); + } + + /** + * Enum for index mode of the underlying composite index + * The default and only index supported right now is star tree index + */ + private enum IndexMode { + STARTREE("startree"); + + private final String typeName; + + IndexMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static IndexMode fromTypeName(String typeName) { + for (IndexMode indexType : IndexMode.values()) { + if (indexType.getTypeName().equalsIgnoreCase(typeName)) { + return indexType; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid index mode in composite index config: [%s] ", typeName)); + } + } + + /** + * Returns the composite fields built based on compositeIndexConfig index settings + */ + public List getCompositeFields() { + return compositeFields; + } + + /** + * Returns whether there are any composite fields as part of the compositeIndexConfig + */ + public boolean hasCompositeFields() { + return !compositeFields.isEmpty(); + } + + /** + * Validates the composite fields based on IndexSettingDefaults and the mappedFieldType + * Updates CompositeIndexConfig with newer, completely updated composite fields + * + */ + public CompositeIndexConfig validateAndGetCompositeIndexConfig( + Function fieldTypeLookup, + CompositeIndexSettings compositeIndexSettings + ) { + if (hasCompositeFields() == false) { + return null; + } + if (!compositeIndexSettings.isEnabled()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index cannot be created, enable it using [%s] setting", + COMPOSITE_INDEX_ENABLED_SETTING.getKey() + ) + ); + } + if (compositeFields.size() > compositeIndexSettings.getMaxFields()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] fields", compositeIndexSettings.getMaxFields()) + ); + } + List validatedAndMappedCompositeFields = new ArrayList<>(); + for (CompositeField compositeField : compositeFields) { + if (compositeField.getDimensionsOrder().size() > compositeIndexSettings.getMaxDimensions()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", compositeIndexSettings.getMaxDimensions()) + ); + } + List dimensions = new ArrayList<>(); + for (Dimension dimension : compositeField.getDimensionsOrder()) { + validateCompositeDimensionField(dimension.getField(), fieldTypeLookup, compositeField.getName()); + dimension = mapDimension(dimension, fieldTypeLookup.apply(dimension.getField())); + dimension.setDefaults(compositeIndexSettings); + dimensions.add(dimension); + } + List metrics = new ArrayList<>(); + for (Metric metric : compositeField.getMetrics()) { + validateCompositeMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); + metric.setDefaults(compositeIndexSettings); + metrics.add(metric); + } + compositeField.getSpec().setDefaults(compositeIndexSettings); + validatedAndMappedCompositeFields.add( + new CompositeField(compositeField.getName(), dimensions, metrics, compositeField.getSpec()) + ); + } + this.compositeFields.clear(); + this.compositeFields.addAll(validatedAndMappedCompositeFields); + return this; + } + + /** + * Maps the dimension to right dimension type based on MappedFieldType + */ + private Dimension mapDimension(Dimension dimension, MappedFieldType fieldType) { + if (!isDimensionMappedToFieldType(dimension, fieldType)) { + return DimensionFactory.createEmptyMappedDimension(dimension, fieldType); + } + return dimension; + } + + /** + * Checks whether dimension field type is same as the source field type + */ + private boolean isDimensionMappedToFieldType(Dimension dimension, MappedFieldType fieldType) { + if (fieldType instanceof DateFieldMapper.DateFieldType) { + return dimension instanceof DateDimension; + } + return true; + } + + /** + * Validations : + * The dimension field should be one of the source fields of the index + * The dimension fields must be aggregation compatible (doc values + field data supported) + * The dimension fields should be of numberField type / dateField type + * + */ + private void validateCompositeDimensionField( + String field, + Function fieldTypeLookup, + String compositeFieldName + ) { + final MappedFieldType ft = fieldTypeLookup.apply(field); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown dimension field [%s] as part of composite field [%s]", field, compositeFieldName) + ); + } + if (!isAllowedDimensionFieldType(ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index is not supported for the dimension field [%s] with field type [%s] as part of " + + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + // doc values not present / field data not supported + if (!ft.isAggregatable()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + } + + /** + * Validations : + * The metric field should be one of the source fields of the index + * The metric fields must be aggregation compatible (doc values + field data supported) + * The metric fields should be of numberField type + * + */ + private void validateCompositeMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { + final MappedFieldType ft = fieldTypeLookup.apply(field); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown metric field [%s] as part of composite field [%s]", field, compositeFieldName) + ); + } + if (!isAllowedMetricFieldType(ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index is not supported for the metric field [%s] with field type [%s] as part of " + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + if (!ft.isAggregatable()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the composite index metric field [%s] with field type [%s] as part of " + + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + } + + private static boolean isAllowedDimensionFieldType(MappedFieldType fieldType) { + return ALLOWED_DIMENSION_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); + } + + private static boolean isAllowedMetricFieldType(MappedFieldType fieldType) { + return ALLOWED_METRIC_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java new file mode 100644 index 0000000000000..59bdfda5469ea --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java @@ -0,0 +1,179 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.util.FeatureFlags; + +import java.util.Arrays; +import java.util.List; + +/** + * Cluster level settings which configures defaults for composite index + */ +@ExperimentalApi +public class CompositeIndexSettings { + /** + * This cluster level setting determines whether composite index is enabled or not + */ + public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( + "indices.composite_index.enabled", + false, + value -> { + if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting determines the max number of composite fields that can be part of composite index config. For each + * composite field, we will generate associated composite index. (eg : star tree index per field ) + */ + public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( + "indices.composite_index.max_fields", + 1, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting determines the max number of dimensions that can be part of composite index field. Number of + * dimensions and associated cardinality has direct effect of composite index size and query performance. + */ + public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( + "indices.composite_index.field.max_dimensions", + 10, + 2, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and + * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa + *

+ * We can remove this later or change it to an enum based constant setting. + * + * @opensearch.experimental + */ + public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( + "indices.composite_index.startree.default.max_leaf_docs", + 10000, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Default intervals for date dimension as part of composite fields + */ + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + "indices.composite_index.field.default.date_intervals", + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + CompositeIndexConfig::getTimeUnit, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( + "indices.composite_index.field.default.metrics", + Arrays.asList( + MetricType.AVG.toString(), + MetricType.COUNT.toString(), + MetricType.SUM.toString(), + MetricType.MAX.toString(), + MetricType.MIN.toString() + ), + MetricType::fromTypeName, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private volatile int maxLeafDocs; + + private volatile List defaultDateIntervals; + + private volatile List defaultMetrics; + private volatile int maxDimensions; + private volatile int maxFields; + private volatile boolean enabled; + + public CompositeIndexSettings(ClusterSettings clusterSettings) { + this.setMaxLeafDocs(clusterSettings.get(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); + this.setDefaultDateIntervals(clusterSettings.get(DEFAULT_DATE_INTERVALS)); + this.setDefaultMetrics(clusterSettings.get(DEFAULT_METRICS_LIST)); + this.setMaxDimensions(clusterSettings.get(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); + this.setMaxFields(clusterSettings.get(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); + this.setEnabled(clusterSettings.get(COMPOSITE_INDEX_ENABLED_SETTING)); + + clusterSettings.addSettingsUpdateConsumer(STAR_TREE_DEFAULT_MAX_LEAF_DOCS, this::setMaxLeafDocs); + clusterSettings.addSettingsUpdateConsumer(DEFAULT_DATE_INTERVALS, this::setDefaultDateIntervals); + clusterSettings.addSettingsUpdateConsumer(DEFAULT_METRICS_LIST, this::setDefaultMetrics); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, this::setMaxDimensions); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_FIELDS_SETTING, this::setMaxFields); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setEnabled); + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public void setMaxLeafDocs(int maxLeafDocs) { + this.maxLeafDocs = maxLeafDocs; + } + + public void setDefaultDateIntervals(List defaultDateIntervals) { + this.defaultDateIntervals = defaultDateIntervals; + } + + public void setDefaultMetrics(List defaultMetrics) { + this.defaultMetrics = defaultMetrics; + } + + public void setMaxDimensions(int maxDimensions) { + this.maxDimensions = maxDimensions; + } + + public void setMaxFields(int maxFields) { + this.maxFields = maxFields; + } + + public int getMaxDimensions() { + return maxDimensions; + } + + public int getMaxFields() { + return maxFields; + } + + public int getMaxLeafDocs() { + return maxLeafDocs; + } + + public boolean isEnabled() { + return enabled; + } + + public List getDefaultDateIntervals() { + return defaultDateIntervals; + } + + public List getDefaultMetrics() { + return defaultMetrics; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java new file mode 100644 index 0000000000000..40145f9f80ef3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; + +/** + * Date dimension class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class DateDimension extends Dimension { + private final List calendarIntervals; + + public DateDimension(String name, Settings settings) { + super(name); + List intervalStrings = settings.getAsList("calendar_interval"); + if (intervalStrings == null || intervalStrings.isEmpty()) { + this.calendarIntervals = new ArrayList<>(); + } else { + this.calendarIntervals = new ArrayList<>(); + for (String interval : intervalStrings) { + this.calendarIntervals.add(CompositeIndexConfig.getTimeUnit(interval)); + } + } + } + + public DateDimension(String name, List calendarIntervals) { + super(name); + this.calendarIntervals = calendarIntervals; + } + + @Override + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (calendarIntervals.isEmpty()) { + this.calendarIntervals.addAll(compositeIndexSettings.getDefaultDateIntervals()); + } + } + + public List getIntervals() { + return calendarIntervals; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java new file mode 100644 index 0000000000000..c06df856d5b8a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Composite index dimension base class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Dimension { + private final String field; + + public Dimension(String field) { + this.field = field; + } + + public String getField() { + return field; + } + + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + // no implementation + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java new file mode 100644 index 0000000000000..dddc3c795078c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; + +/** + * Holds details of metrics field as part of composite field + */ +@ExperimentalApi +public class Metric { + private final String field; + private final List metrics; + + public Metric(String field, List metrics) { + this.field = field; + this.metrics = metrics; + } + + public String getField() { + return field; + } + + public List getMetrics() { + return metrics; + } + + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (metrics.isEmpty()) { + metrics.addAll(compositeIndexSettings.getDefaultMetrics()); + } + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java new file mode 100644 index 0000000000000..9261279fa5957 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Supported metric types for composite index + */ +@ExperimentalApi +public enum MetricType { + COUNT("count"), + AVG("avg"), + SUM("sum"), + MIN("min"), + MAX("max"); + + private final String typeName; + + MetricType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static MetricType fromTypeName(String typeName) { + for (MetricType metric : MetricType.values()) { + if (metric.getTypeName().equalsIgnoreCase(typeName)) { + return metric; + } + } + throw new IllegalArgumentException("Invalid metric type: " + typeName); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java new file mode 100644 index 0000000000000..c38f8ca8813c7 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Star tree index specific settings for a composite field. + */ +@ExperimentalApi +public class StarTreeFieldSpec implements CompositeFieldSpec { + + private final AtomicInteger maxLeafDocs = new AtomicInteger(); + private final List skipStarNodeCreationInDims; + private final StarTreeBuildMode buildMode; + + public StarTreeFieldSpec(int maxLeafDocs, List skipStarNodeCreationInDims, StarTreeBuildMode buildMode) { + this.maxLeafDocs.set(maxLeafDocs); + this.skipStarNodeCreationInDims = skipStarNodeCreationInDims; + this.buildMode = buildMode; + } + + /** + * Star tree build mode using which sorting and aggregations are performed during index creation. + * + * @opensearch.experimental + */ + @ExperimentalApi + public enum StarTreeBuildMode { + ON_HEAP("onheap"), + OFF_HEAP("offheap"); + + private final String typeName; + + StarTreeBuildMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static StarTreeBuildMode fromTypeName(String typeName) { + for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) { + if (starTreeBuildMode.getTypeName().equalsIgnoreCase(typeName)) { + return starTreeBuildMode; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", typeName)); + } + } + + public int maxLeafDocs() { + return maxLeafDocs.get(); + } + + @Override + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (maxLeafDocs.get() == Integer.MAX_VALUE) { + maxLeafDocs.set(compositeIndexSettings.getMaxLeafDocs()); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java new file mode 100644 index 0000000000000..59f18efec26b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Core classes for handling composite indices. + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex; diff --git a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java new file mode 100644 index 0000000000000..be49269f0264c --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.compositeindex.CompositeIndexSettings; + +/** + * Utility to provide a {@link CompositeIndexSettings} instance containing all defaults + * + * @opensearch.experimental + */ +@ExperimentalApi +public final class DefaultCompositeIndexSettings { + private DefaultCompositeIndexSettings() {} + + public static final CompositeIndexSettings INSTANCE = new CompositeIndexSettings( + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 251be8a990055..b1fb2f77e2981 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -106,6 +106,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.cache.request.ShardRequestCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineConfigFactory; @@ -354,6 +355,7 @@ public class IndicesService extends AbstractLifecycleComponent private final BiFunction translogFactorySupplier; private volatile TimeValue clusterDefaultRefreshInterval; private final SearchRequestStats searchRequestStats; + private final CompositeIndexSettings compositeIndexSettings; @Override protected void doStart() { @@ -388,7 +390,8 @@ public IndicesService( @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, RecoverySettings recoverySettings, CacheService cacheService, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) { this.settings = settings; this.threadPool = threadPool; @@ -495,6 +498,7 @@ protected void closeInternal() { .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate); this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; + this.compositeIndexSettings = compositeIndexSettings; } /** @@ -903,7 +907,8 @@ private synchronized IndexService createIndexService( translogFactorySupplier, this::getClusterDefaultRefreshInterval, this.recoverySettings, - this.remoteStoreSettings + this.remoteStoreSettings, + this.compositeIndexSettings ); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 9462aeddbd0e4..949cc2982ad6f 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -146,6 +146,7 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationStatsTracker; import org.opensearch.index.analysis.AnalysisRegistry; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; @@ -833,6 +834,8 @@ protected Node( final SearchRequestStats searchRequestStats = new SearchRequestStats(clusterService.getClusterSettings()); final SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService); + final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterService.getClusterSettings()); + remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); CacheModule cacheModule = new CacheModule(pluginsService.filterPlugins(CachePlugin.class), settings); CacheService cacheService = cacheModule.getCacheService(); @@ -863,7 +866,8 @@ protected Node( remoteStoreStatsTrackerFactory, recoverySettings, cacheService, - remoteStoreSettings + remoteStoreSettings, + compositeIndexSettings ); final IngestService ingestService = new IngestService( diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 3d6a54055d3d5..d5269252681a7 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -1118,6 +1118,7 @@ private IndicesService mockIndicesServices(DocumentMapper documentMapper) throws when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() { }); when(indexService.getIndexSortSupplier()).thenReturn(() -> null); + when(indexService.getCompositeIndexConfigSupplier()).thenReturn(() -> null); // noinspection unchecked return ((CheckedFunction) invocationOnMock.getArguments()[1]).apply(indexService); }); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 4ce4936c047d9..8f45a872e752c 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -99,6 +99,7 @@ import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.TranslogFactory; +import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesQueryCache; @@ -264,7 +265,8 @@ private IndexService newIndexService(IndexModule module) throws IOException { translogFactorySupplier, () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + DefaultCompositeIndexSettings.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java new file mode 100644 index 0000000000000..98a017bf1acbd --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java @@ -0,0 +1,233 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.opensearch.index.IndexSettingsTests.newIndexMeta; + +/** + * Composite index config settings unit tests + */ +public class CompositeIndexConfigSettingsTests extends OpenSearchTestCase { + private static IndexSettings indexSettings(Settings settings) { + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); + } + + public void testDefaultSettings() { + Settings settings = Settings.EMPTY; + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertFalse(compositeIndexConfig.hasCompositeFields()); + } + + public void testMinimumMetrics() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("metrics is required for composite index field [my_field]", exception.getMessage()); + } + + public void testMinimumDimensions() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Atleast two dimensions are required to build composite index field [my_field]", exception.getMessage()); + } + + public void testInvalidDimensionType() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Invalid dimension type in composite index config: [invalid] ", exception.getMessage()); + } + + public void testInvalidIndexMode() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .put("index.composite_index.config.my_field.index_mode", "invalid") + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Invalid index mode in composite index config: [invalid] ", exception.getMessage()); + } + + public void testValidCompositeIndexConfig() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + } + + public void testCompositeIndexMultipleFields() { + Settings settings = Settings.builder() + .put("indices.composite_index.max_fields", 2) + .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.field1.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.field2.dimensions_order", Arrays.asList("dim3", "dim4")) + .put("index.composite_index.config.field2.dimensions_config.dim3.field", "dim3_field") + .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") + .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(2, compositeIndexConfig.getCompositeFields().size()); + } + + public void testCompositeIndexDateIntervalsSetting() { + Settings settings = Settings.builder() + .putList("indices.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "date") + .putList("index.composite_index.config.my_field.dimensions_config.dim1.calendar_interval", Arrays.asList("day", "week")) + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + List expectedIntervals = Arrays.asList( + Rounding.DateTimeUnit.DAY_OF_MONTH, + Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR + ); + assertEquals(expectedIntervals, ((DateDimension) compositeField.getDimensionsOrder().get(0)).getIntervals()); + } + + public void testCompositeIndexMetricsSetting() { + Settings settings = Settings.builder() + .putList("indices.composite_index.field.default.metrics", Arrays.asList("count", "max")) + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.my_field.metrics_config.metric1.metrics", Arrays.asList("count", "max")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + List expectedMetrics = Arrays.asList(MetricType.COUNT, MetricType.MAX); + assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + } + + public void testValidateWithoutCompositeSettingEnabled() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "default") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.LONG)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric2", NumberFieldMapper.NumberType.LONG)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterSettings); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, compositeIndexSettings) + ); + assertEquals( + "composite index cannot be created, enable it using [indices.composite_index.enabled] setting", + exception.getMessage() + ); + + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + } + + public void testEnabledWithFFOff() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(testWithEnabledSettings(settings)); + Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); + ClusterSettings clusterSettings = new ClusterSettings(settings1, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new CompositeIndexSettings(clusterSettings) + ); + assertEquals( + "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + exception.getMessage() + ); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + assertTrue(compositeField.getDimensionsOrder().get(0) instanceof Dimension); + assertTrue(compositeField.getDimensionsOrder().get(1) instanceof DateDimension); + } + + private IndexSettings createIndexSettings(Settings settings) { + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); + } + + public IndexSettings testWithEnabledSettings(Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings1), Settings.EMPTY); + return indexSettings; + } + +} diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 8f1d58cf201e9..a59e061330b91 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -218,6 +218,7 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() { }); when(indexService.getIndexSortSupplier()).thenReturn(() -> null); + when(indexService.getCompositeIndexConfigSupplier()).thenReturn(() -> null); // noinspection unchecked return ((CheckedFunction) invocationOnMock.getArguments()[1]).apply(indexService); }); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 460aaa08a224d..1b44c2c3008d9 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -193,6 +193,7 @@ import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheStats; +import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesService; @@ -2086,7 +2087,8 @@ public void onFailure(final Exception e) { new RemoteStoreStatsTrackerFactory(clusterService, settings), DefaultRecoverySettings.INSTANCE, new CacheModule(new ArrayList<>(), settings).getCacheService(), - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + DefaultCompositeIndexSettings.INSTANCE ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( From dce81944397d9aa11f67b7648ff3b0ff57dc31df Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 6 Jun 2024 14:48:05 +0530 Subject: [PATCH 2/2] Adding integ settings and correcting the flow to contain only enabled setting at cluster level Signed-off-by: Bharathwaj G --- .../common/settings/ClusterSettings.java | 10 +- .../common/settings/IndexScopedSettings.java | 12 +- .../org/opensearch/index/IndexModule.java | 5 +- .../org/opensearch/index/IndexService.java | 6 +- .../compositeindex/CompositeFieldSpec.java | 4 +- .../compositeindex/CompositeIndexConfig.java | 445 +++++++++++------ .../CompositeIndexSettings.java | 179 ------- .../index/compositeindex/DateDimension.java | 16 +- .../index/compositeindex/Dimension.java | 4 - .../index/compositeindex/Metric.java | 4 +- .../compositeindex/StarTreeFieldSpec.java | 7 - .../DefaultCompositeIndexSettings.java | 28 -- .../opensearch/indices/IndicesService.java | 39 +- .../main/java/org/opensearch/node/Node.java | 6 +- .../opensearch/index/IndexModuleTests.java | 3 +- .../CompositeIndexConfigIntegTests.java | 236 +++++++++ .../CompositeIndexConfigSettingsTests.java | 472 ++++++++++++++++-- .../snapshots/SnapshotResiliencyTests.java | 4 +- 18 files changed, 1023 insertions(+), 457 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java delete mode 100644 server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 3b01b1c4fd7b7..abd015de2a9c8 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -114,7 +114,6 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.remote.RemoteStorePressureSettings; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.store.remote.filecache.FileCacheSettings; @@ -748,13 +747,8 @@ public void apply(Settings value, Settings current, Settings previous) { SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, - // Composite index settings - CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING, - CompositeIndexSettings.COMPOSITE_INDEX_MAX_FIELDS_SETTING, - CompositeIndexSettings.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, - CompositeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, - CompositeIndexSettings.DEFAULT_METRICS_LIST, - CompositeIndexSettings.DEFAULT_DATE_INTERVALS + // Composite index setting + IndicesService.COMPOSITE_INDEX_ENABLED_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 38e93b22806ea..814e50b99ae24 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -51,6 +51,7 @@ import org.opensearch.index.SearchSlowLog; import org.opensearch.index.TieredMergePolicyProvider; import org.opensearch.index.cache.bitset.BitsetFilterCache; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.FieldMapper; @@ -238,6 +239,14 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for concurrent segment search IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING, IndexSettings.ALLOW_DERIVED_FIELDS, + + // Settings for composite index defaults + CompositeIndexConfig.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, + CompositeIndexConfig.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, + CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING, + CompositeIndexConfig.DEFAULT_METRICS_LIST, + CompositeIndexConfig.DEFAULT_DATE_INTERVALS, + // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { Map groups = s.getAsGroups(); @@ -250,7 +259,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { } }, Property.IndexScope), // this allows similarity settings to be passed Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed - Setting.groupSetting("index.composite_index.", Property.IndexScope) // this allows composite index settings to be passed + Setting.groupSetting("index.composite_index.config.", Property.IndexScope) // this allows composite index settings to be + // passed ) ) ); diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index aaec6bfec2123..18122745a4218 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -66,7 +66,6 @@ import org.opensearch.index.cache.query.DisabledQueryCache; import org.opensearch.index.cache.query.IndexQueryCache; import org.opensearch.index.cache.query.QueryCache; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -608,7 +607,7 @@ public IndexService newIndexService( Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -668,7 +667,7 @@ public IndexService newIndexService( clusterDefaultRefreshIntervalSupplier, recoverySettings, remoteStoreSettings, - compositeIndexSettings + isCompositeIndexCreationEnabled ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 6f994e8f67c5b..96081388baae5 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -73,7 +73,6 @@ import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.query.QueryCache; import org.opensearch.index.compositeindex.CompositeIndexConfig; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -227,7 +226,7 @@ public IndexService( Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -267,8 +266,9 @@ public IndexService( } if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) { + // The validation is done right after the merge of the mapping later in the process ( similar to sort ) this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig() - .validateAndGetCompositeIndexConfig(mapperService::fieldType, compositeIndexSettings); + .validateAndGetCompositeIndexConfig(mapperService::fieldType, isCompositeIndexCreationEnabled); } else { this.compositeIndexConfigSupplier = () -> null; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java index 51a2b454abcd9..829084ebf8b1a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java @@ -17,6 +17,4 @@ */ @ExperimentalApi -public interface CompositeFieldSpec { - void setDefaults(CompositeIndexSettings compositeIndexSettings); -} +public interface CompositeFieldSpec {} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java index 308c4b62d3996..a785d7adb6557 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java @@ -10,23 +10,26 @@ import org.opensearch.common.Rounding; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.indices.IndicesService; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.BooleanSupplier; import java.util.function.Function; -import static org.opensearch.index.compositeindex.CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING; - /** * Configuration of composite index containing list of composite fields. * Each composite field contains dimensions, metrics along with composite index (eg: star tree) specific settings. @@ -45,6 +48,76 @@ public class CompositeIndexConfig { NumberFieldMapper.NumberFieldType.class ); + /** + * This setting determines the max number of composite fields that can be part of composite index config. For each + * composite field, we will generate associated composite index. (eg : star tree index per field ) + */ + public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( + "index.composite_index.max_fields", + 1, + 1, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting determines the max number of dimensions that can be part of composite index field. Number of + * dimensions and associated cardinality has direct effect of composite index size and query performance. + */ + public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( + "index.composite_index.field.max_dimensions", + 10, + 2, + 10, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and + * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa + *

+ * We can remove this later or change it to an enum based constant setting. + * + * @opensearch.experimental + */ + public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( + "index.composite_index.startree.default.max_leaf_docs", + 10000, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * Default intervals for date dimension as part of composite fields + */ + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + "index.composite_index.field.default.date_intervals", + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + CompositeIndexConfig::getTimeUnit, + Setting.Property.IndexScope, + Setting.Property.Final + ); + public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( + "index.composite_index.field.default.metrics", + Arrays.asList( + MetricType.AVG.toString(), + MetricType.COUNT.toString(), + MetricType.SUM.toString(), + MetricType.MAX.toString(), + MetricType.MIN.toString() + ), + MetricType::fromTypeName, + Setting.Property.IndexScope, + Setting.Property.Final + ); + private volatile int maxLeafDocs; + private volatile List defaultDateIntervals; + private volatile List defaultMetrics; + private volatile int maxDimensions; + private volatile int maxFields; private static final String COMPOSITE_INDEX_CONFIG = "index.composite_index.config"; private static final String DIMENSIONS_ORDER = "dimensions_order"; private static final String DIMENSIONS_CONFIG = "dimensions_config"; @@ -58,23 +131,57 @@ public class CompositeIndexConfig { private static final String SKIP_STAR_NODE_CREATION_FOR_DIMS = "skip_star_node_creation_for_dimensions"; private static final String SPEC = "_spec"; private final List compositeFields = new ArrayList<>(); + private final IndexSettings indexSettings; public CompositeIndexConfig(IndexSettings indexSettings) { - + this.setMaxLeafDocs(indexSettings.getValue(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); + this.setDefaultDateIntervals(indexSettings.getValue(DEFAULT_DATE_INTERVALS)); + this.setDefaultMetrics(indexSettings.getValue(DEFAULT_METRICS_LIST)); + this.setMaxDimensions(indexSettings.getValue(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); + this.setMaxFields(indexSettings.getValue(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); final Map compositeIndexSettings = indexSettings.getSettings().getGroups(COMPOSITE_INDEX_CONFIG); + this.indexSettings = indexSettings; Set fields = compositeIndexSettings.keySet(); + if (!fields.isEmpty()) { + if (!FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING)) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + if (fields.size() > getMaxFields()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] fields", getMaxFields()) + ); + } + } for (String field : fields) { compositeFields.add(buildCompositeField(field, compositeIndexSettings.get(field))); } } /** - * This returns composite field after performing basic validations and doesn't do field type validations etc + * This returns composite field after performing basic validations but doesn't perform field type based validations * */ private CompositeField buildCompositeField(String field, Settings compositeFieldSettings) { + + List dimensions = buildDimensions(field, compositeFieldSettings); + List metrics = buildMetrics(field, compositeFieldSettings); + List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); + IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); + Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); + CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder, dimensions, this); + return new CompositeField(field, dimensions, metrics, compositeFieldSpec); + } + + /** + * Returns dimensions after performing validations + * + */ + private List buildDimensions(String field, Settings compositeFieldSettings) { List dimensions = new ArrayList<>(); - List metrics = new ArrayList<>(); List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); if (dimensionsOrder == null || dimensionsOrder.isEmpty()) { throw new IllegalArgumentException( @@ -87,6 +194,12 @@ private CompositeField buildCompositeField(String field, Settings compositeField ); } + if (dimensionsOrder.size() > getMaxDimensions()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", getMaxDimensions()) + ); + } + Map dimConfig = compositeFieldSettings.getGroups(DIMENSIONS_CONFIG); for (String dimension : dimConfig.keySet()) { @@ -102,25 +215,6 @@ private CompositeField buildCompositeField(String field, Settings compositeField } } - List metricFields = compositeFieldSettings.getAsList(METRICS); - if (metricFields == null || metricFields.isEmpty()) { - throw new IllegalArgumentException(String.format(Locale.ROOT, "metrics is required for composite index field [%s]", field)); - } - Map metricsConfig = compositeFieldSettings.getGroups(METRICS_CONFIG); - - for (String metricField : metricsConfig.keySet()) { - if (!metricFields.contains(metricField)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "metric field [%s] is not present in 'metrics' for composite index field [%s]", - metricField, - field - ) - ); - } - } - Set uniqueDimensions = new HashSet<>(); for (String dimension : dimensionsOrder) { if (!uniqueDimensions.add(dimension)) { @@ -133,72 +227,16 @@ private CompositeField buildCompositeField(String field, Settings compositeField ) ); } - dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension))); + dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension), this)); } - uniqueDimensions = null; - Set uniqueMetricFields = new HashSet<>(); - for (String metricField : metricFields) { - if (!uniqueMetricFields.add(metricField)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "duplicate metric field [%s] found in 'metrics' for composite index field [%s]", - metricField, - field - ) - ); - } - Settings metricSettings = metricsConfig.get(metricField); - if (metricSettings == null) { - // fill cluster level defaults in create flow as part of CompositeIndexSupplier - metrics.add(new Metric(metricField, new ArrayList<>())); - } else { - String name = metricSettings.get(FIELD, metricField); - List metricsList = metricSettings.getAsList(METRICS); - if (metricsList.isEmpty()) { - // fill cluster level defaults in create flow as part of CompositeIndexSupplier - metrics.add(new Metric(name, new ArrayList<>())); - } else { - List metricTypes = new ArrayList<>(); - Set uniqueMetricTypes = new HashSet<>(); - for (String metric : metricsList) { - if (!uniqueMetricTypes.add(metric)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "duplicate metric type [%s] found in metrics for composite index field [%s]", - metric, - field - ) - ); - } - metricTypes.add(MetricType.fromTypeName(metric)); - } - uniqueMetricTypes = null; - metrics.add(new Metric(name, metricTypes)); - } - } - } - uniqueMetricFields = null; - - IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); - Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); - CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder); - return new CompositeField(field, dimensions, metrics, compositeFieldSpec); - } - - public static Rounding.DateTimeUnit getTimeUnit(String expression) { - if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { - throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); - } - return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + return dimensions; } /** * Dimension factory based on field type */ private static class DimensionFactory { - static Dimension create(String dimension, Settings settings) { + static Dimension create(String dimension, Settings settings, CompositeIndexConfig compositeIndexConfig) { if (settings == null) { return new Dimension(dimension); } @@ -208,20 +246,13 @@ static Dimension create(String dimension, Settings settings) { case DEFAULT: return new Dimension(field); case DATE: - return new DateDimension(field, settings); + return new DateDimension(field, settings, compositeIndexConfig); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid dimension type [%s] in composite index config", type) ); } } - - static Dimension createEmptyMappedDimension(Dimension dimension, MappedFieldType type) { - if (type instanceof DateFieldMapper.DateFieldType) { - return new DateDimension(dimension.getField(), new ArrayList<>()); - } - return dimension; - } } /** @@ -254,17 +285,94 @@ public static DimensionType fromTypeName(String typeName) { } } + /** + * Returns metrics after performing validations + * + */ + private List buildMetrics(String field, Settings compositeFieldSettings) { + List metrics = new ArrayList<>(); + List metricFields = compositeFieldSettings.getAsList(METRICS); + if (metricFields == null || metricFields.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "metrics is required for composite index field [%s]", field)); + } + Map metricsConfig = compositeFieldSettings.getGroups(METRICS_CONFIG); + + for (String metricField : metricsConfig.keySet()) { + if (!metricFields.contains(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "metric field [%s] is not present in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + } + Set uniqueMetricFields = new HashSet<>(); + for (String metricField : metricFields) { + if (!uniqueMetricFields.add(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric field [%s] found in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + Settings metricSettings = metricsConfig.get(metricField); + if (metricSettings == null) { + metrics.add(new Metric(metricField, getDefaultMetrics())); + } else { + String name = metricSettings.get(FIELD, metricField); + List metricsList = metricSettings.getAsList(METRICS); + if (metricsList.isEmpty()) { + metrics.add(new Metric(name, getDefaultMetrics())); + } else { + List metricTypes = new ArrayList<>(); + Set uniqueMetricTypes = new HashSet<>(); + for (String metric : metricsList) { + if (!uniqueMetricTypes.add(metric)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric type [%s] found in metrics for composite index field [%s]", + metric, + field + ) + ); + } + metricTypes.add(MetricType.fromTypeName(metric)); + } + metrics.add(new Metric(name, metricTypes)); + } + } + } + return metrics; + } + /** * Composite field spec factory based on index mode */ private static class CompositeFieldSpecFactory { - static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + static CompositeFieldSpec create( + IndexMode indexMode, + Settings settings, + List dimensionsOrder, + List dimensions, + CompositeIndexConfig compositeIndexConfig + ) { if (settings == null) { - return new StarTreeFieldSpec(10000, new ArrayList<>(), StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP); + return new StarTreeFieldSpec( + compositeIndexConfig.getMaxLeafDocs(), + new ArrayList<>(), + StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP + ); } switch (indexMode) { case STARTREE: - return buildStarTreeFieldSpec(settings, dimensions); + return buildStarTreeFieldSpec(settings, dimensionsOrder, dimensions, compositeIndexConfig); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid index mode [%s] in composite index config", indexMode) @@ -273,26 +381,37 @@ static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + private static StarTreeFieldSpec buildStarTreeFieldSpec( + Settings settings, + List dimensionsString, + List dimensions, + CompositeIndexConfig compositeIndexConfig + ) { StarTreeFieldSpec.StarTreeBuildMode buildMode = StarTreeFieldSpec.StarTreeBuildMode.fromTypeName( settings.get(STAR_TREE_BUILD_MODE, StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP.getTypeName()) ); - // Fill default value as part of create flow as part of supplier - int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, Integer.MAX_VALUE); + int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, compositeIndexConfig.getMaxLeafDocs()); if (maxLeafDocs < 1) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid max_leaf_docs [%s] in composite index config", maxLeafDocs) ); } List skipStarNodeCreationInDims = settings.getAsList(SKIP_STAR_NODE_CREATION_FOR_DIMS, new ArrayList<>()); + Set skipListWithMappedFieldNames = new HashSet<>(); for (String dim : skipStarNodeCreationInDims) { - if (!dimensions.contains(dim)) { + if (!dimensionsString.contains(dim)) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid dimension [%s] in skip_star_node_creation_for_dims", dim) ); } + boolean duplicate = !(skipListWithMappedFieldNames.add(dimensions.get(dimensionsString.indexOf(dim)).getField())); + if (duplicate) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "duplicate dimension [%s] found in skipStarNodeCreationInDims", dim) + ); + } } - return new StarTreeFieldSpec(maxLeafDocs, skipStarNodeCreationInDims, buildMode); + return new StarTreeFieldSpec(maxLeafDocs, new ArrayList<>(skipListWithMappedFieldNames), buildMode); } /** @@ -337,71 +456,37 @@ public boolean hasCompositeFields() { } /** - * Validates the composite fields based on IndexSettingDefaults and the mappedFieldType - * Updates CompositeIndexConfig with newer, completely updated composite fields + * Validates the composite fields based on defaults and based on the mappedFieldType + * Updates CompositeIndexConfig with newer, completely updated composite fields. * */ public CompositeIndexConfig validateAndGetCompositeIndexConfig( Function fieldTypeLookup, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) { if (hasCompositeFields() == false) { return null; } - if (!compositeIndexSettings.isEnabled()) { + if (!isCompositeIndexCreationEnabled.getAsBoolean()) { throw new IllegalArgumentException( String.format( Locale.ROOT, "composite index cannot be created, enable it using [%s] setting", - COMPOSITE_INDEX_ENABLED_SETTING.getKey() + IndicesService.COMPOSITE_INDEX_ENABLED_SETTING.getKey() ) ); } - if (compositeFields.size() > compositeIndexSettings.getMaxFields()) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "composite index can have atmost [%s] fields", compositeIndexSettings.getMaxFields()) - ); - } - List validatedAndMappedCompositeFields = new ArrayList<>(); for (CompositeField compositeField : compositeFields) { - if (compositeField.getDimensionsOrder().size() > compositeIndexSettings.getMaxDimensions()) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", compositeIndexSettings.getMaxDimensions()) - ); - } - List dimensions = new ArrayList<>(); for (Dimension dimension : compositeField.getDimensionsOrder()) { - validateCompositeDimensionField(dimension.getField(), fieldTypeLookup, compositeField.getName()); - dimension = mapDimension(dimension, fieldTypeLookup.apply(dimension.getField())); - dimension.setDefaults(compositeIndexSettings); - dimensions.add(dimension); + validateDimensionField(dimension, fieldTypeLookup, compositeField.getName()); } - List metrics = new ArrayList<>(); for (Metric metric : compositeField.getMetrics()) { - validateCompositeMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); - metric.setDefaults(compositeIndexSettings); - metrics.add(metric); + validateMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); } - compositeField.getSpec().setDefaults(compositeIndexSettings); - validatedAndMappedCompositeFields.add( - new CompositeField(compositeField.getName(), dimensions, metrics, compositeField.getSpec()) - ); } - this.compositeFields.clear(); - this.compositeFields.addAll(validatedAndMappedCompositeFields); return this; } - /** - * Maps the dimension to right dimension type based on MappedFieldType - */ - private Dimension mapDimension(Dimension dimension, MappedFieldType fieldType) { - if (!isDimensionMappedToFieldType(dimension, fieldType)) { - return DimensionFactory.createEmptyMappedDimension(dimension, fieldType); - } - return dimension; - } - /** * Checks whether dimension field type is same as the source field type */ @@ -419,15 +504,27 @@ private boolean isDimensionMappedToFieldType(Dimension dimension, MappedFieldTyp * The dimension fields should be of numberField type / dateField type * */ - private void validateCompositeDimensionField( - String field, - Function fieldTypeLookup, - String compositeFieldName - ) { - final MappedFieldType ft = fieldTypeLookup.apply(field); + private void validateDimensionField(Dimension dimension, Function fieldTypeLookup, String compositeFieldName) { + final MappedFieldType ft = fieldTypeLookup.apply(dimension.getField()); if (ft == null) { throw new IllegalArgumentException( - String.format(Locale.ROOT, "unknown dimension field [%s] as part of composite field [%s]", field, compositeFieldName) + String.format( + Locale.ROOT, + "unknown dimension field [%s] as part of composite field [%s]", + dimension.getField(), + compositeFieldName + ) + ); + } + if (!isDimensionMappedToFieldType(dimension, ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "specify field type [%s] for dimension field [%s] as part of of composite field [%s]", + ft.typeName(), + dimension.getField(), + compositeFieldName + ) ); } if (!isAllowedDimensionFieldType(ft)) { @@ -436,7 +533,7 @@ private void validateCompositeDimensionField( Locale.ROOT, "composite index is not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", - field, + dimension.getField(), ft.typeName(), compositeFieldName ) @@ -448,7 +545,7 @@ private void validateCompositeDimensionField( String.format( Locale.ROOT, "Aggregations not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", - field, + dimension.getField(), ft.typeName(), compositeFieldName ) @@ -463,7 +560,7 @@ private void validateCompositeDimensionField( * The metric fields should be of numberField type * */ - private void validateCompositeMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { + private void validateMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { final MappedFieldType ft = fieldTypeLookup.apply(field); if (ft == null) { throw new IllegalArgumentException( @@ -502,4 +599,52 @@ private static boolean isAllowedDimensionFieldType(MappedFieldType fieldType) { private static boolean isAllowedMetricFieldType(MappedFieldType fieldType) { return ALLOWED_METRIC_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); } + + public static Rounding.DateTimeUnit getTimeUnit(String expression) { + if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); + } + return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + } + + public void setMaxLeafDocs(int maxLeafDocs) { + this.maxLeafDocs = maxLeafDocs; + } + + public void setDefaultDateIntervals(List defaultDateIntervals) { + this.defaultDateIntervals = defaultDateIntervals; + } + + public void setDefaultMetrics(List defaultMetrics) { + this.defaultMetrics = defaultMetrics; + } + + public void setMaxDimensions(int maxDimensions) { + this.maxDimensions = maxDimensions; + } + + public void setMaxFields(int maxFields) { + this.maxFields = maxFields; + } + + public int getMaxDimensions() { + return maxDimensions; + } + + public int getMaxFields() { + return maxFields; + } + + public int getMaxLeafDocs() { + return maxLeafDocs; + } + + public List getDefaultDateIntervals() { + return defaultDateIntervals; + } + + public List getDefaultMetrics() { + return defaultMetrics; + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java deleted file mode 100644 index 59bdfda5469ea..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex; - -import org.opensearch.common.Rounding; -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.util.FeatureFlags; - -import java.util.Arrays; -import java.util.List; - -/** - * Cluster level settings which configures defaults for composite index - */ -@ExperimentalApi -public class CompositeIndexSettings { - /** - * This cluster level setting determines whether composite index is enabled or not - */ - public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( - "indices.composite_index.enabled", - false, - value -> { - if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { - throw new IllegalArgumentException( - "star tree index is under an experimental feature and can be activated only by enabling " - + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() - + " feature flag in the JVM options" - ); - } - }, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting determines the max number of composite fields that can be part of composite index config. For each - * composite field, we will generate associated composite index. (eg : star tree index per field ) - */ - public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( - "indices.composite_index.max_fields", - 1, - 1, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting determines the max number of dimensions that can be part of composite index field. Number of - * dimensions and associated cardinality has direct effect of composite index size and query performance. - */ - public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( - "indices.composite_index.field.max_dimensions", - 10, - 2, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and - * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa - *

- * We can remove this later or change it to an enum based constant setting. - * - * @opensearch.experimental - */ - public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( - "indices.composite_index.startree.default.max_leaf_docs", - 10000, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Default intervals for date dimension as part of composite fields - */ - public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( - "indices.composite_index.field.default.date_intervals", - Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), - CompositeIndexConfig::getTimeUnit, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( - "indices.composite_index.field.default.metrics", - Arrays.asList( - MetricType.AVG.toString(), - MetricType.COUNT.toString(), - MetricType.SUM.toString(), - MetricType.MAX.toString(), - MetricType.MIN.toString() - ), - MetricType::fromTypeName, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private volatile int maxLeafDocs; - - private volatile List defaultDateIntervals; - - private volatile List defaultMetrics; - private volatile int maxDimensions; - private volatile int maxFields; - private volatile boolean enabled; - - public CompositeIndexSettings(ClusterSettings clusterSettings) { - this.setMaxLeafDocs(clusterSettings.get(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); - this.setDefaultDateIntervals(clusterSettings.get(DEFAULT_DATE_INTERVALS)); - this.setDefaultMetrics(clusterSettings.get(DEFAULT_METRICS_LIST)); - this.setMaxDimensions(clusterSettings.get(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); - this.setMaxFields(clusterSettings.get(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); - this.setEnabled(clusterSettings.get(COMPOSITE_INDEX_ENABLED_SETTING)); - - clusterSettings.addSettingsUpdateConsumer(STAR_TREE_DEFAULT_MAX_LEAF_DOCS, this::setMaxLeafDocs); - clusterSettings.addSettingsUpdateConsumer(DEFAULT_DATE_INTERVALS, this::setDefaultDateIntervals); - clusterSettings.addSettingsUpdateConsumer(DEFAULT_METRICS_LIST, this::setDefaultMetrics); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, this::setMaxDimensions); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_FIELDS_SETTING, this::setMaxFields); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setEnabled); - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public void setMaxLeafDocs(int maxLeafDocs) { - this.maxLeafDocs = maxLeafDocs; - } - - public void setDefaultDateIntervals(List defaultDateIntervals) { - this.defaultDateIntervals = defaultDateIntervals; - } - - public void setDefaultMetrics(List defaultMetrics) { - this.defaultMetrics = defaultMetrics; - } - - public void setMaxDimensions(int maxDimensions) { - this.maxDimensions = maxDimensions; - } - - public void setMaxFields(int maxFields) { - this.maxFields = maxFields; - } - - public int getMaxDimensions() { - return maxDimensions; - } - - public int getMaxFields() { - return maxFields; - } - - public int getMaxLeafDocs() { - return maxLeafDocs; - } - - public boolean isEnabled() { - return enabled; - } - - public List getDefaultDateIntervals() { - return defaultDateIntervals; - } - - public List getDefaultMetrics() { - return defaultMetrics; - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java index 40145f9f80ef3..3e88759ec56b9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java @@ -24,11 +24,11 @@ public class DateDimension extends Dimension { private final List calendarIntervals; - public DateDimension(String name, Settings settings) { + public DateDimension(String name, Settings settings, CompositeIndexConfig compositeIndexConfig) { super(name); List intervalStrings = settings.getAsList("calendar_interval"); if (intervalStrings == null || intervalStrings.isEmpty()) { - this.calendarIntervals = new ArrayList<>(); + this.calendarIntervals = compositeIndexConfig.getDefaultDateIntervals(); } else { this.calendarIntervals = new ArrayList<>(); for (String interval : intervalStrings) { @@ -37,18 +37,6 @@ public DateDimension(String name, Settings settings) { } } - public DateDimension(String name, List calendarIntervals) { - super(name); - this.calendarIntervals = calendarIntervals; - } - - @Override - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - if (calendarIntervals.isEmpty()) { - this.calendarIntervals.addAll(compositeIndexSettings.getDefaultDateIntervals()); - } - } - public List getIntervals() { return calendarIntervals; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java index c06df856d5b8a..a18ffcd1df0db 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java @@ -26,8 +26,4 @@ public Dimension(String field) { public String getField() { return field; } - - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - // no implementation - } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java index dddc3c795078c..9467cf1176f7a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java @@ -33,9 +33,9 @@ public List getMetrics() { return metrics; } - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + public void setDefaults(CompositeIndexConfig compositeIndexConfig) { if (metrics.isEmpty()) { - metrics.addAll(compositeIndexSettings.getDefaultMetrics()); + metrics.addAll(compositeIndexConfig.getDefaultMetrics()); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java index c38f8ca8813c7..c3ac54feeb5e4 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java @@ -63,11 +63,4 @@ public static StarTreeBuildMode fromTypeName(String typeName) { public int maxLeafDocs() { return maxLeafDocs.get(); } - - @Override - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - if (maxLeafDocs.get() == Integer.MAX_VALUE) { - maxLeafDocs.set(compositeIndexSettings.getMaxLeafDocs()); - } - } } diff --git a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java deleted file mode 100644 index be49269f0264c..0000000000000 --- a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.indices; - -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.compositeindex.CompositeIndexSettings; - -/** - * Utility to provide a {@link CompositeIndexSettings} instance containing all defaults - * - * @opensearch.experimental - */ -@ExperimentalApi -public final class DefaultCompositeIndexSettings { - private DefaultCompositeIndexSettings() {} - - public static final CompositeIndexSettings INSTANCE = new CompositeIndexSettings( - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ); -} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b1fb2f77e2981..e087b0d76977b 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -74,6 +74,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.OpenSearchExecutors; @@ -106,7 +107,6 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.cache.request.ShardRequestCache; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineConfigFactory; @@ -307,6 +307,25 @@ public class IndicesService extends AbstractLifecycleComponent Property.Final ); + /** + * This cluster level setting determines whether composite index is enabled or not + */ + public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( + "indices.composite_index.enabled", + false, + value -> { + if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** * The node's settings. */ @@ -355,7 +374,7 @@ public class IndicesService extends AbstractLifecycleComponent private final BiFunction translogFactorySupplier; private volatile TimeValue clusterDefaultRefreshInterval; private final SearchRequestStats searchRequestStats; - private final CompositeIndexSettings compositeIndexSettings; + private volatile boolean compositeIndexCreationEnabled; @Override protected void doStart() { @@ -390,8 +409,7 @@ public IndicesService( @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, RecoverySettings recoverySettings, CacheService cacheService, - RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + RemoteStoreSettings remoteStoreSettings ) { this.settings = settings; this.threadPool = threadPool; @@ -443,6 +461,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setCompositeIndexCreationEnabled); // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -498,7 +518,6 @@ protected void closeInternal() { .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate); this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; - this.compositeIndexSettings = compositeIndexSettings; } /** @@ -908,7 +927,7 @@ private synchronized IndexService createIndexService( this::getClusterDefaultRefreshInterval, this.recoverySettings, this.remoteStoreSettings, - this.compositeIndexSettings + this::isCompositeIndexCreationEnabled ); } @@ -1901,6 +1920,14 @@ private void setIdFieldDataEnabled(boolean value) { this.idFieldDataEnabled = value; } + private void setCompositeIndexCreationEnabled(boolean value) { + this.compositeIndexCreationEnabled = value; + } + + public boolean isCompositeIndexCreationEnabled() { + return compositeIndexCreationEnabled; + } + private void updateDanglingIndicesInfo(Index index) { assert DiscoveryNode.isDataNode(settings) : "dangling indices information should only be persisted on data nodes"; assert nodeWriteDanglingIndicesInfo : "writing dangling indices info is not enabled"; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 23a9740601a8e..cb1f2caa082fc 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -147,7 +147,6 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationStatsTracker; import org.opensearch.index.analysis.AnalysisRegistry; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; @@ -839,8 +838,6 @@ protected Node( final SearchRequestStats searchRequestStats = new SearchRequestStats(clusterService.getClusterSettings()); final SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService); - final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterService.getClusterSettings()); - remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); CacheModule cacheModule = new CacheModule(pluginsService.filterPlugins(CachePlugin.class), settings); CacheService cacheService = cacheModule.getCacheService(); @@ -871,8 +868,7 @@ protected Node( remoteStoreStatsTrackerFactory, recoverySettings, cacheService, - remoteStoreSettings, - compositeIndexSettings + remoteStoreSettings ); final IngestService ingestService = new IngestService( diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 8f45a872e752c..829e65569bc75 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -99,7 +99,6 @@ import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.TranslogFactory; -import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesQueryCache; @@ -266,7 +265,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, DefaultRemoteStoreSettings.INSTANCE, - DefaultCompositeIndexSettings.INSTANCE + () -> false ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java new file mode 100644 index 0000000000000..6145be8887c85 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java @@ -0,0 +1,236 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexService; +import org.opensearch.indices.IndicesService; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class CompositeIndexConfigIntegTests extends OpenSearchIntegTestCase { + + private static final XContentBuilder TEST_MAPPING = createTestMapping(); + + private static XContentBuilder createTestMapping() { + try { + return jsonBuilder().startObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric") + .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") + .field("doc_values", true) + .endObject() + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.COMPOSITE_INDEX, "true").build(); + } + + @Before + public final void setupNodeSettings() { + Settings request = Settings.builder().put(IndicesService.COMPOSITE_INDEX_ENABLED_SETTING.getKey(), true).build(); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get()); + } + + @After + public final void cleanupNodeSettings() { + assertAcked( + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("*")) + .setTransientSettings(Settings.builder().putNull("*")) + ); + } + + public void testInvalidCompositeIndex() { + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("specify field type [date] for dimension field [timestamp] as part of of composite field [my_field]", ex.getMessage()); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "Aggregations not supported for the composite index metric field [numeric] with field type [integer] as part of composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("invalid", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("unknown dimension field [invalid] as part of composite field [my_field]", ex.getMessage()); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("invalid")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("unknown metric field [invalid] as part of composite field [my_field]", ex.getMessage()); + + FeatureFlagSetter.set(FeatureFlags.COMPOSITE_INDEX); + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "2") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get(); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("test"); + GetSettingsResponse indexSettings = client().admin().indices().getSettings(getSettingsRequest).actionGet(); + indexSettings.getIndexToSettings().get("test"); + final Index index = resolveIndex("test"); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + CompositeIndexConfig compositeIndexConfig = indexService.getIndexSettings().getCompositeIndexConfig(); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("timestamp", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("numeric_dv", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0) instanceof DateDimension); + } + } + } + + public void testValidCompositeIndex() { + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "2") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get(); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("test"); + GetSettingsResponse indexSettings = client().admin().indices().getSettings(getSettingsRequest).actionGet(); + indexSettings.getIndexToSettings().get("test"); + final Index index = resolveIndex("test"); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + CompositeIndexConfig compositeIndexConfig = indexService.getIndexSettings().getCompositeIndexConfig(); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("timestamp", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("numeric_dv", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0) instanceof DateDimension); + List expectedMetrics = Arrays.asList( + MetricType.AVG, + MetricType.COUNT, + MetricType.SUM, + MetricType.MAX, + MetricType.MIN + ); + assertEquals(expectedMetrics, compositeIndexConfig.getCompositeFields().get(0).getMetrics().get(0).getMetrics()); + List expectedIntervals = Arrays.asList( + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY + ); + assertEquals( + expectedIntervals, + ((DateDimension) compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0)).getIntervals() + ); + } + } + + } + +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java index 98a017bf1acbd..ef87345df2017 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java @@ -9,43 +9,51 @@ package org.opensearch.index.compositeindex; import org.opensearch.common.Rounding; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.IpFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.BooleanSupplier; import java.util.function.Function; +import static org.opensearch.common.util.FeatureFlags.COMPOSITE_INDEX; import static org.opensearch.index.IndexSettingsTests.newIndexMeta; /** * Composite index config settings unit tests */ public class CompositeIndexConfigSettingsTests extends OpenSearchTestCase { - private static IndexSettings indexSettings(Settings settings) { - return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); - } public void testDefaultSettings() { Settings settings = Settings.EMPTY; - IndexSettings indexSettings = indexSettings(settings); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertFalse(compositeIndexConfig.hasCompositeFields()); } public void testMinimumMetrics() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + Settings settings = Settings.builder() .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + ; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("metrics is required for composite index field [my_field]", exception.getMessage()); + + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testMinimumDimensions() { @@ -55,8 +63,11 @@ public void testMinimumDimensions() { .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Atleast two dimensions are required to build composite index field [my_field]", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testInvalidDimensionType() { @@ -66,8 +77,11 @@ public void testInvalidDimensionType() { .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Invalid dimension type in composite index config: [invalid] ", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testInvalidIndexMode() { @@ -78,8 +92,31 @@ public void testInvalidIndexMode() { .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .put("index.composite_index.config.my_field.index_mode", "invalid") .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Invalid index mode in composite index config: [invalid] ", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDefaultNumberofCompositeFieldsValidation() { + Settings settings = Settings.builder() + .put("index.composite_index.max_fields", 1) + .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.field1.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.field2.dimensions_order", Arrays.asList("dim3", "dim4")) + .put("index.composite_index.config.field2.dimensions_config.dim3.field", "dim3_field") + .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") + .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); + assertEquals("composite index can have atmost [1] fields", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testValidCompositeIndexConfig() { @@ -87,17 +124,25 @@ public void testValidCompositeIndexConfig() { .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .put("index.composite_index.config.my_field.index_mode", "startree") .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("dim1_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("dim2_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1) instanceof DateDimension); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexMultipleFields() { Settings settings = Settings.builder() - .put("indices.composite_index.max_fields", 2) + .put("index.composite_index.max_fields", 2) .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") @@ -107,15 +152,33 @@ public void testCompositeIndexMultipleFields() { .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + Exception ex = expectThrows(IllegalArgumentException.class, () -> getEnabledAndMultiFieldIndexSettings(settings)); + assertEquals("Failed to parse value [2] for setting [index.composite_index.max_fields] must be <= 1", ex.getMessage()); + /** + * // uncomment once we add support for multiple fields + IndexSettings indexSettings = getEnabledAndMultiFieldIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(2, compositeIndexConfig.getCompositeFields().size()); + assertEquals(2, compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().size()); + assertEquals(2, compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().size()); + assertEquals(1, compositeIndexConfig.getCompositeFields().get(0).getMetrics().size()); + assertEquals(1, compositeIndexConfig.getCompositeFields().get(1).getMetrics().size()); + assertEquals("dim1_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("dim2_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertEquals("dim3_field", compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().get(0).getField()); + assertEquals("dim4_field", compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().get(1).getField()); + assertEquals("metric1", compositeIndexConfig.getCompositeFields().get(0).getMetrics().get(0).getField()); + assertEquals("metric2", compositeIndexConfig.getCompositeFields().get(1).getMetrics().get(0).getField()); + **/ + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexDateIntervalsSetting() { Settings settings = Settings.builder() - .putList("indices.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) + .putList("index.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "date") @@ -123,7 +186,9 @@ public void testCompositeIndexDateIntervalsSetting() { .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); @@ -133,27 +198,31 @@ public void testCompositeIndexDateIntervalsSetting() { Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR ); assertEquals(expectedIntervals, ((DateDimension) compositeField.getDimensionsOrder().get(0)).getIntervals()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexMetricsSetting() { Settings settings = Settings.builder() - .putList("indices.composite_index.field.default.metrics", Arrays.asList("count", "max")) + .putList("index.composite_index.field.default.metrics", Arrays.asList("count", "max")) .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .putList("index.composite_index.config.my_field.metrics_config.metric1.metrics", Arrays.asList("count", "max")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); List expectedMetrics = Arrays.asList(MetricType.COUNT, MetricType.MAX); assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } - public void testValidateWithoutCompositeSettingEnabled() { + public void testCompositeIndexEnabledSetting() { Settings settings = Settings.builder() .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") @@ -162,29 +231,25 @@ public void testValidateWithoutCompositeSettingEnabled() { .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) .build(); - + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); Map fieldTypes = new HashMap<>(); fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.LONG)); fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric2", NumberFieldMapper.NumberType.LONG)); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); Function fieldTypeLookup = fieldTypes::get; - - CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterSettings); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, compositeIndexSettings) + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getDisabledSupplier()) ); assertEquals( "composite index cannot be created, enable it using [indices.composite_index.enabled] setting", exception.getMessage() ); - - assertTrue(compositeIndexConfig.hasCompositeFields()); - assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testEnabledWithFFOff() { @@ -200,34 +265,363 @@ public void testEnabledWithFFOff() { fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new CompositeIndexConfig(getEnabledIndexSettings(settings)) + ); + assertEquals( + "star tree index is under an experimental feature and can be activated only by enabling " + + "opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + exception.getMessage() + ); + } + + public void testUnknownDimField() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + + Map fieldTypes = new HashMap<>(); + + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals("unknown dimension field [dim1_field] as part of composite field [my_field]", exception.getMessage()); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testUnknownMetricField() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals("unknown metric field [metric2] as part of composite field [my_field]", exception.getMessage()); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionMappedType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new IpFieldMapper.IpFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + Function fieldTypeLookup = fieldTypes::get; - CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(testWithEnabledSettings(settings)); - Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); - ClusterSettings clusterSettings = new ClusterSettings(settings1, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> new CompositeIndexSettings(clusterSettings) + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) ); assertEquals( - "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + "composite index is not supported for the dimension field [dim1_field] with field type [ip] as part of composite field [my_field]", exception.getMessage() ); - assertTrue(compositeIndexConfig.hasCompositeFields()); - assertEquals(1, compositeIndexConfig.getCompositeFields().size()); - CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); - assertTrue(compositeField.getDimensionsOrder().get(0) instanceof Dimension); - assertTrue(compositeField.getDimensionsOrder().get(1) instanceof DateDimension); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDimsWithNoDocValues() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put( + "dim1_field", + new NumberFieldMapper.NumberFieldType( + "dim1_field", + NumberFieldMapper.NumberType.LONG, + false, + false, + false, + true, + null, + Collections.emptyMap() + ) + ); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "Aggregations not supported for the dimension field [dim1_field] with field type [long] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testMetricsWithNoDocValues() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put( + "metric1", + new NumberFieldMapper.NumberFieldType( + "metric1", + NumberFieldMapper.NumberType.LONG, + false, + false, + false, + true, + null, + Collections.emptyMap() + ) + ); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "Aggregations not supported for the composite index metric field [metric1] with field type [long] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionMappedKeywordType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new KeywordFieldMapper.KeywordFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "composite index is not supported for the dimension field [dim1_field] with " + + "field type [keyword] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidMetricMappedKeywordType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new DateFieldMapper.DateFieldType("metric1")); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "composite index is not supported for the metric field [metric1] with field type [date] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDefaults() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + CompositeIndexConfig config = compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()); + + assertTrue(config.hasCompositeFields()); + assertEquals(1, config.getCompositeFields().size()); + CompositeField compositeField = config.getCompositeFields().get(0); + List expectedMetrics = Arrays.asList(MetricType.AVG, MetricType.COUNT, MetricType.SUM, MetricType.MAX, MetricType.MIN); + assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + StarTreeFieldSpec spec = (StarTreeFieldSpec) compositeField.getSpec(); + assertEquals(10000, spec.maxLeafDocs()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDimTypeValidation() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "specify field type [date] for dimension field [dim2_field] as part of of composite field [my_field]", + ex.getMessage() + ); } private IndexSettings createIndexSettings(Settings settings) { return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); } - public IndexSettings testWithEnabledSettings(Settings settings) { - Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); - IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings1), Settings.EMPTY); + private Settings getEnabledSettings() { + return Settings.builder().put("index.composite_index.enabled", true).build(); + } + + public IndexSettings getEnabledIndexSettings(Settings settings) { + Settings enabledSettings = Settings.builder().put(settings).put("index.composite_index.enabled", true).build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", enabledSettings), getEnabledSettings()); return indexSettings; } + public IndexSettings getEnabledAndMultiFieldIndexSettings(Settings settings) { + Settings multiFieldEnabledSettings = Settings.builder() + .put(settings) + .put("index.composite_index.enabled", true) + .put(CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING.getKey(), 2) + .build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", multiFieldEnabledSettings), multiFieldEnabledSettings); + return indexSettings; + } + + private BooleanSupplier getEnabledSupplier() { + return new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + return true; + } + }; + } + + private BooleanSupplier getDisabledSupplier() { + return new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + return false; + } + }; + } } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index e21c9860d0c46..86de008b5dee5 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -192,7 +192,6 @@ import org.opensearch.index.shard.PrimaryReplicaSyncer; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCacheStats; -import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesService; @@ -2085,8 +2084,7 @@ public void onFailure(final Exception e) { new RemoteStoreStatsTrackerFactory(clusterService, settings), DefaultRecoverySettings.INSTANCE, new CacheModule(new ArrayList<>(), settings).getCacheService(), - DefaultRemoteStoreSettings.INSTANCE, - DefaultCompositeIndexSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService(