diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java index 5073858848e05..fe21cd2b6acf1 100644 --- a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java +++ b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java @@ -178,7 +178,8 @@ private StringTerms newTerms(Random rand, BytesRef[] dict, boolean withNested) { 0, buckets, 0, - new TermsAggregator.BucketCountThresholds(1, 0, topNSize, numShards) + new TermsAggregator.BucketCountThresholds(1, 0, topNSize, numShards), + false ); } diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java index b98a257dfbf48..402dd3d3241d8 100644 --- a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java +++ b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java @@ -94,7 +94,8 @@ private StringTerms newTerms(boolean withNested) { 100000, resultBuckets, 0, - new TermsAggregator.BucketCountThresholds(1, 0, buckets, buckets) + new TermsAggregator.BucketCountThresholds(1, 0, buckets, buckets), + false ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsFixedDocCountErrorIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsFixedDocCountErrorIT.java new file mode 100644 index 0000000000000..8ca32c568f12f --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsFixedDocCountErrorIT.java @@ -0,0 +1,301 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.bucket; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.aggregations.AggregationBuilders.terms; +import static org.opensearch.test.OpenSearchIntegTestCase.Scope.TEST; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +@OpenSearchIntegTestCase.ClusterScope(scope = TEST, numClientNodes = 0, maxNumDataNodes = 1, supportsDedicatedMasters = false) +public class TermsFixedDocCountErrorIT extends ParameterizedOpenSearchIntegTestCase { + + private static final String STRING_FIELD_NAME = "s_value"; + + public TermsFixedDocCountErrorIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + + public void testSimpleAggErrorMultiShard() throws Exception { + // size = 2, shard_size = 2 + // Shard_1 [A, A, A, A, B, B, C] -> Buckets {"A" : 4, "B" : 2} + // Shard_2 [A, B, B, B, C, C] -> Buckets {"B" : 3, "C" : 2} + // coordinator -> Buckets {"B" : 5, "A" : 4} + // Agg error is 4, from (shard_size)th bucket on each shard + // Bucket "A" error is 2, from (shard_size)th bucket on shard_2 + // Bucket "B" error is 0, it's present on both shards + + assertAcked( + prepareCreate("idx_mshard_1").setMapping(STRING_FIELD_NAME, "type=keyword") + .setSettings( + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + ); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + refresh("idx_mshard_1"); + + IndicesSegmentResponse segmentResponse = client().admin().indices().prepareSegments("idx_mshard_1").get(); + assertEquals(1, segmentResponse.getIndices().get("idx_mshard_1").getShards().get(0).getShards()[0].getSegments().size()); + + assertAcked( + prepareCreate("idx_mshard_2").setMapping(STRING_FIELD_NAME, "type=keyword") + .setSettings( + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + ); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + refresh("idx_mshard_2"); + + segmentResponse = client().admin().indices().prepareSegments("idx_mshard_2").get(); + assertEquals(1, segmentResponse.getIndices().get("idx_mshard_2").getShards().get(0).getShards()[0].getSegments().size()); + + SearchResponse response = client().prepareSearch("idx_mshard_2", "idx_mshard_1") + .setSize(0) + .addAggregation(terms("terms").field(STRING_FIELD_NAME).showTermDocCountError(true).size(2).shardSize(2)) + .get(); + + Terms terms = response.getAggregations().get("terms"); + assertEquals(2, terms.getBuckets().size()); + assertEquals(4, terms.getDocCountError()); + + Terms.Bucket bucket = terms.getBuckets().get(0); // Bucket "B" + assertEquals("B", bucket.getKey().toString()); + assertEquals(5, bucket.getDocCount()); + assertEquals(0, bucket.getDocCountError()); + + bucket = terms.getBuckets().get(1); // Bucket "A" + assertEquals("A", bucket.getKey().toString()); + assertEquals(4, bucket.getDocCount()); + assertEquals(2, bucket.getDocCountError()); + } + + // 1 shard 1 segment case + public void testSimpleAggErrorSingleShard() throws Exception { + assertAcked( + prepareCreate("idx_shard_error").setMapping(STRING_FIELD_NAME, "type=keyword") + .setSettings( + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + ); + client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + refresh("idx_shard_error"); + + SearchResponse response = client().prepareSearch("idx_shard_error") + .setSize(0) + .addAggregation(terms("terms").field(STRING_FIELD_NAME).showTermDocCountError(true).size(1).shardSize(2)) + .get(); + + Terms terms = response.getAggregations().get("terms"); + assertEquals(1, terms.getBuckets().size()); + assertEquals(0, terms.getDocCountError()); + + Terms.Bucket bucket = terms.getBuckets().get(0); // Bucket "B" + assertEquals("A", bucket.getKey().toString()); + assertEquals(6, bucket.getDocCount()); + assertEquals(0, bucket.getDocCountError()); + } + + public void testSliceLevelDocCountErrorSingleShard() throws Exception { + assumeTrue( + "Slice level error is not relevant to non-concurrent search cases", + internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) + ); + + // Slices are created by sorting segments by doc count in descending order then distributing in round robin fashion. + // Tries to create segments (and therefore slices since slice_count = 2) as follows: + // 1. [A, A, A, B, B, C] + // 2. [A, B, B, B, C, C] + // Thus we expect the doc count error for A to be 2 as the nth largest bucket on slice 2 has size 2 + + assertAcked( + prepareCreate("idx_slice_error").setMapping(STRING_FIELD_NAME, "type=keyword") + .setSettings( + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + ); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + refresh("idx_slice_error"); + + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + refresh("idx_slice_error"); + + IndicesSegmentResponse segmentResponse = client().admin().indices().prepareSegments("idx_slice_error").get(); + assertEquals(2, segmentResponse.getIndices().get("idx_slice_error").getShards().get(0).getShards()[0].getSegments().size()); + + // Confirm that there is no error when shard_size == slice_size > cardinality + SearchResponse response = client().prepareSearch("idx_slice_error") + .setSize(0) + .addAggregation(terms("terms").field(STRING_FIELD_NAME).showTermDocCountError(true).size(1).shardSize(4)) + .get(); + + Terms terms = response.getAggregations().get("terms"); + assertEquals(1, terms.getBuckets().size()); + assertEquals(0, terms.getDocCountError()); + + Terms.Bucket bucket = terms.getBuckets().get(0); // Bucket "B" + assertEquals("B", bucket.getKey().toString()); + assertEquals(5, bucket.getDocCount()); + assertEquals(0, bucket.getDocCountError()); + + response = client().prepareSearch("idx_slice_error") + .setSize(0) + .addAggregation(terms("terms").field(STRING_FIELD_NAME).showTermDocCountError(true).size(2).shardSize(2)) + .get(); + + terms = response.getAggregations().get("terms"); + assertEquals(2, terms.getBuckets().size()); + assertEquals(4, terms.getDocCountError()); + + bucket = terms.getBuckets().get(0); // Bucket "B" + assertEquals("B", bucket.getKey().toString()); + assertEquals(5, bucket.getDocCount()); + assertEquals(0, bucket.getDocCountError()); + + bucket = terms.getBuckets().get(1); // Bucket "A" + assertEquals("A", bucket.getKey().toString()); + assertEquals(3, bucket.getDocCount()); + assertEquals(2, bucket.getDocCountError()); + } + + public void testSliceLevelDocCountErrorMultiShard() throws Exception { + assumeTrue( + "Slice level error is not relevant to non-concurrent search cases", + internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) + ); + + // Size = 2, shard_size = 2 + // Shard_1 [A, A, A, A, B, B, C, C] + // slice_1 [A, A, A, B, B, C] {"A" : 3, "B" : 2} + // slice_2 [A, C] {"A" : 1, "C" : 1} + // shard buckets: {"A" : 4 - error: 0, "B" : 2 - error: 1} + // Shard_2 [A, A, B, B, B, C, C, C] + // slice_1 [A, B, B, B, C, C] {"B" : 3, "C" : 2} + // slice_2 [A, C] {"A" : 1, "C" : 1} + // shard buckets: {"B" : 3 - error: 1, "C" : 3 - error: 0} + // Overall + // {"B" : 5 - error: 2, "A" : 4 - error: 3} Agg error: 6 + + assertAcked( + prepareCreate("idx_mshard_1").setMapping(STRING_FIELD_NAME, "type=keyword") + .setSettings( + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + ); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + refresh("idx_mshard_1"); + + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + refresh("idx_mshard_1"); + + IndicesSegmentResponse segmentResponse = client().admin().indices().prepareSegments("idx_mshard_1").get(); + assertEquals(2, segmentResponse.getIndices().get("idx_mshard_1").getShards().get(0).getShards()[0].getSegments().size()); + + assertAcked( + prepareCreate("idx_mshard_2").setMapping(STRING_FIELD_NAME, "type=keyword") + .setSettings( + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + ); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "B").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + refresh("idx_mshard_2"); + + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get(); + client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "C").endObject()).get(); + refresh("idx_mshard_2"); + + segmentResponse = client().admin().indices().prepareSegments("idx_mshard_2").get(); + assertEquals(2, segmentResponse.getIndices().get("idx_mshard_2").getShards().get(0).getShards()[0].getSegments().size()); + + SearchResponse response = client().prepareSearch("idx_mshard_2", "idx_mshard_1") + .setSize(0) + .addAggregation(terms("terms").field(STRING_FIELD_NAME).showTermDocCountError(true).size(2).shardSize(2)) + .get(); + + Terms terms = response.getAggregations().get("terms"); + assertEquals(2, terms.getBuckets().size()); + assertEquals(6, terms.getDocCountError()); + + Terms.Bucket bucket = terms.getBuckets().get(0); // Bucket "B" + assertEquals("B", bucket.getKey().toString()); + assertEquals(5, bucket.getDocCount()); + assertEquals(2, bucket.getDocCountError()); + + bucket = terms.getBuckets().get(1); // Bucket "A" + assertEquals("A", bucket.getKey().toString()); + assertEquals(4, bucket.getDocCount()); + assertEquals(3, bucket.getDocCountError()); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsShardMinDocCountIT.java index b0d8e7ea02e8f..ea96befb03bd3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsShardMinDocCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsShardMinDocCountIT.java @@ -131,7 +131,7 @@ public void testShardMinDocCountSignificantTermsTest() throws Exception { (filter("inclass", QueryBuilders.termQuery("class", true))).subAggregation( significantTerms("mySignificantTerms").field("text") .minDocCount(2) - .shardSize(2) + .shardSize(10) .shardMinDocCount(2) .size(2) .executionHint(randomExecutionHint()) @@ -198,7 +198,7 @@ public void testShardMinDocCountTermsTest() throws Exception { .minDocCount(2) .shardMinDocCount(2) .size(2) - .shardSize(2) + .shardSize(10) .executionHint(randomExecutionHint()) .order(BucketOrder.key(true)) ) diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java index d06a0ed9976fc..f61740bce34a8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java @@ -83,7 +83,8 @@ protected StringTerms buildEmptyTermsAggregation() { 0, emptyList(), 0, - bucketCountThresholds + bucketCountThresholds, + false ); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/DoubleTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/DoubleTerms.java index de02d5a938644..9fce2abbdd383 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/DoubleTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/DoubleTerms.java @@ -137,7 +137,8 @@ public DoubleTerms( long otherDocCount, List buckets, long docCountError, - TermsAggregator.BucketCountThresholds bucketCountThresholds + TermsAggregator.BucketCountThresholds bucketCountThresholds, + boolean hasSliceLevelDocCountError ) { super( name, @@ -150,7 +151,8 @@ public DoubleTerms( otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } @@ -179,7 +181,8 @@ public DoubleTerms create(List buckets) { otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } @@ -196,7 +199,14 @@ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) } @Override - protected DoubleTerms create(String name, List buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) { + protected DoubleTerms create( + String name, + List buckets, + BucketOrder reduceOrder, + long docCountError, + long otherDocCount, + boolean hasSliceLevelDocCountError + ) { return new DoubleTerms( name, reduceOrder, @@ -208,7 +218,8 @@ protected DoubleTerms create(String name, List buckets, BucketOrder redu otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index 5ed899408ab40..9d30700d80054 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -808,7 +808,8 @@ StringTerms buildResult(long owningBucketOrd, long otherDocCount, StringTerms.Bu otherDocCount, Arrays.asList(topBuckets), 0, - bucketCountThresholds + bucketCountThresholds, + false ); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedTerms.java index d542064df24d7..7093c504f0ed2 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedTerms.java @@ -71,9 +71,10 @@ protected InternalMappedTerms( long otherDocCount, List buckets, long docCountError, - TermsAggregator.BucketCountThresholds bucketCountThresholds + TermsAggregator.BucketCountThresholds bucketCountThresholds, + boolean hasSliceLevelDocCountError ) { - super(name, reduceOrder, order, bucketCountThresholds, metadata); + super(name, reduceOrder, order, bucketCountThresholds, metadata, hasSliceLevelDocCountError); this.format = format; this.shardSize = shardSize; this.showTermDocCountError = showTermDocCountError; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java index 5b90163fa3959..d3d205ea9440a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java @@ -242,7 +242,7 @@ public InternalMultiTerms( List buckets, TermsAggregator.BucketCountThresholds bucketCountThresholds ) { - super(name, reduceOrder, order, bucketCountThresholds, metadata); + super(name, reduceOrder, order, bucketCountThresholds, metadata, false); this.shardSize = shardSize; this.showTermDocCountError = showTermDocCountError; this.otherDocCount = otherDocCount; @@ -349,7 +349,8 @@ protected InternalMultiTerms create( List buckets, BucketOrder reduceOrder, long docCountError, - long otherDocCount + long otherDocCount, + boolean hasSliceLevelDocCountError ) { return new InternalMultiTerms( name, diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java index 0e773291881cf..0d280f5748235 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java @@ -225,21 +225,25 @@ public int hashCode() { protected final int requiredSize; protected final long minDocCount; protected final TermsAggregator.BucketCountThresholds bucketCountThresholds; + protected boolean hasSliceLevelDocCountError = false; /** * Creates a new {@link InternalTerms} - * @param name The name of the aggregation - * @param reduceOrder The {@link BucketOrder} that should be used to merge shard results. - * @param order The {@link BucketOrder} that should be used to sort the final reduce. - * @param bucketCountThresholds Object containing values for minDocCount, shardMinDocCount, size, shardSize. - * @param metadata The metadata associated with the aggregation. + * + * @param name The name of the aggregation + * @param reduceOrder The {@link org.opensearch.search.aggregations.BucketOrder} that should be used to merge shard results. + * @param order The {@link org.opensearch.search.aggregations.BucketOrder} that should be used to sort the final reduce. + * @param bucketCountThresholds Object containing values for minDocCount, shardMinDocCount, size, shardSize. + * @param metadata The metadata associated with the aggregation. + * @param hasSliceLevelDocCountError */ protected InternalTerms( String name, BucketOrder reduceOrder, BucketOrder order, TermsAggregator.BucketCountThresholds bucketCountThresholds, - Map metadata + Map metadata, + boolean hasSliceLevelDocCountError ) { super(name, metadata); this.reduceOrder = reduceOrder; @@ -247,6 +251,7 @@ protected InternalTerms( this.bucketCountThresholds = bucketCountThresholds; this.requiredSize = bucketCountThresholds.getRequiredSize(); this.minDocCount = bucketCountThresholds.getMinDocCount(); + this.hasSliceLevelDocCountError = hasSliceLevelDocCountError; } /** @@ -299,9 +304,8 @@ private BucketOrder getReduceOrder(List aggregations) { private long getDocCountError(InternalTerms terms, ReduceContext reduceContext) { int size = terms.getBuckets().size(); - // doc_count_error is always computed at the coordinator based on the buckets returned by the shards. This should be 0 during the - // shard level reduce as no buckets are being pruned at this stage. - if (reduceContext.isSliceLevel() || size == 0 || size < terms.getShardSize() || isKeyOrder(terms.order)) { + // TODO: I think this can be size <= terms.getShardSize() but need to validate + if (size == 0 || size < terms.getShardSize() || isKeyOrder(terms.order)) { return 0; } else if (InternalOrder.isCountDesc(terms.order)) { if (terms.getDocCountError() > 0) { @@ -309,6 +313,10 @@ private long getDocCountError(InternalTerms terms, ReduceContext reduceCon // use this as the error for this aggregation return terms.getDocCountError(); } else { + // We need a way to indicate to the coordinator that doc count error was gathered at the slice level, so do that here + if (reduceContext.isSliceLevel()) { + hasSliceLevelDocCountError = true; + } // otherwise use the doc count of the last term in the // aggregation return terms.getBuckets().stream().mapToLong(MultiBucketsAggregation.Bucket::getDocCount).min().getAsLong(); @@ -500,14 +508,35 @@ For backward compatibility, we disable the merge sort and use ({@link InternalTe if (sumDocCountError == -1) { docCountError = -1; } else { - docCountError = aggregations.size() == 1 ? 0 : sumDocCountError; + // If there is doc count error originating from slice_size that needs to be handled differently: + // If there is slice level doc count error then that needs to be propagated to the top level doc count error even if no + // additional error is introduced by shard_size -- in other words the 1 shard case + // However, if there is only 1 slice, then we can set the doc count error to 0 and disregard any slice level doc count error, + // which is what the shards did before. + if (reduceContext.isFinalReduce() && hasSliceLevelDocCountError) { + docCountError = sumDocCountError; + } else { + if (aggregations.size() == 1) { + docCountError = 0; + hasSliceLevelDocCountError = false; + } else { + docCountError = sumDocCountError; + } + } } // Shards must return buckets sorted by key, so we apply the sort here in shard level reduce if (reduceContext.isSliceLevel()) { Arrays.sort(list, thisReduceOrder.comparator()); } - return create(name, Arrays.asList(list), reduceContext.isFinalReduce() ? order : thisReduceOrder, docCountError, otherDocCount); + return create( + name, + Arrays.asList(list), + reduceContext.isFinalReduce() ? order : thisReduceOrder, + docCountError, + otherDocCount, + hasSliceLevelDocCountError + ); } @Override @@ -523,7 +552,7 @@ protected B reduceBucket(List buckets, ReduceContext context) { for (B bucket : buckets) { docCount += bucket.getDocCount(); if (docCountError != -1) { - if (bucket.showDocCountError() == false || bucket.getDocCountError() == -1) { + if (bucket.showDocCountError() == false) { docCountError = -1; } else { docCountError += bucket.getDocCountError(); @@ -539,7 +568,14 @@ protected B reduceBucket(List buckets, ReduceContext context) { protected abstract int getShardSize(); - protected abstract A create(String name, List buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount); + protected abstract A create( + String name, + List buckets, + BucketOrder reduceOrder, + long docCountError, + long otherDocCount, + boolean hasSliceLevelDocCountError + ); /** * Create an array to hold some buckets. Used in collecting the results. diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongTerms.java index fe78145dce3e7..49d29dc89e55d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongTerms.java @@ -149,7 +149,8 @@ public LongTerms( long otherDocCount, List buckets, long docCountError, - TermsAggregator.BucketCountThresholds bucketCountThresholds + TermsAggregator.BucketCountThresholds bucketCountThresholds, + boolean hasSliceLevelDocCountError ) { super( name, @@ -162,7 +163,8 @@ public LongTerms( otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } @@ -191,7 +193,8 @@ public LongTerms create(List buckets) { otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } @@ -208,7 +211,14 @@ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) } @Override - protected LongTerms create(String name, List buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) { + protected LongTerms create( + String name, + List buckets, + BucketOrder reduceOrder, + long docCountError, + long otherDocCount, + boolean hasSliceLevelDocCountError + ) { return new LongTerms( name, reduceOrder, @@ -220,7 +230,8 @@ protected LongTerms create(String name, List buckets, BucketOrder reduce otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } @@ -296,7 +307,8 @@ static DoubleTerms convertLongTermsToDouble(LongTerms longTerms, DocValueFormat longTerms.otherDocCount, newBuckets, longTerms.docCountError, - longTerms.bucketCountThresholds + longTerms.bucketCountThresholds, + longTerms.hasSliceLevelDocCountError ); } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java index ade23f7290f89..83a77bcc157b1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java @@ -463,7 +463,8 @@ StringTerms buildResult(long owningBucketOrd, long otherDocCount, StringTerms.Bu otherDocCount, Arrays.asList(topBuckets), 0, - bucketCountThresholds + bucketCountThresholds, + false ); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/NumericTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/NumericTermsAggregator.java index 9d095bbf7dccf..b7125c7d3a6de 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/NumericTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/NumericTermsAggregator.java @@ -404,7 +404,8 @@ LongTerms buildResult(long owningBucketOrd, long otherDocCount, LongTerms.Bucket otherDocCount, List.of(topBuckets), 0, - bucketCountThresholds + bucketCountThresholds, + false ); } @@ -421,7 +422,8 @@ LongTerms buildEmptyResult() { 0, emptyList(), 0, - bucketCountThresholds + bucketCountThresholds, + false ); } } @@ -484,7 +486,8 @@ DoubleTerms buildResult(long owningBucketOrd, long otherDocCount, DoubleTerms.Bu otherDocCount, List.of(topBuckets), 0, - bucketCountThresholds + bucketCountThresholds, + false ); } @@ -501,7 +504,8 @@ DoubleTerms buildEmptyResult() { 0, emptyList(), 0, - bucketCountThresholds + bucketCountThresholds, + false ); } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringTerms.java index 6dedc65ff14e3..112e811ab0dbc 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringTerms.java @@ -141,7 +141,8 @@ public StringTerms( long otherDocCount, List buckets, long docCountError, - TermsAggregator.BucketCountThresholds bucketCountThresholds + TermsAggregator.BucketCountThresholds bucketCountThresholds, + boolean hasSliceLevelDocCountError ) { super( name, @@ -154,7 +155,8 @@ public StringTerms( otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } @@ -183,7 +185,8 @@ public StringTerms create(List buckets) { otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } @@ -205,7 +208,14 @@ Bucket createBucket(long docCount, InternalAggregations aggs, long docCountError } @Override - protected StringTerms create(String name, List buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) { + protected StringTerms create( + String name, + List buckets, + BucketOrder reduceOrder, + long docCountError, + long otherDocCount, + boolean hasSliceLevelDocCountError + ) { return new StringTerms( name, reduceOrder, @@ -217,7 +227,8 @@ protected StringTerms create(String name, List buckets, BucketOrder redu otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + hasSliceLevelDocCountError ); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedTerms.java index 3d2bbb93c889a..500cff0a14d55 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedTerms.java @@ -78,7 +78,7 @@ public UnmappedTerms( TermsAggregator.BucketCountThresholds bucketCountThresholds, Map metadata ) { - super(name, order, order, bucketCountThresholds, metadata); + super(name, order, order, bucketCountThresholds, metadata, false); } /** @@ -119,7 +119,14 @@ Bucket createBucket(long docCount, InternalAggregations aggs, long docCountError } @Override - protected UnmappedTerms create(String name, List buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) { + protected UnmappedTerms create( + String name, + List buckets, + BucketOrder reduceOrder, + long docCountError, + long otherDocCount, + boolean hasSliceLevelDocCountError + ) { throw new UnsupportedOperationException("not supported for UnmappedTerms"); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnsignedLongTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnsignedLongTerms.java index edeec00d366fd..3e10132ff55ad 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnsignedLongTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnsignedLongTerms.java @@ -141,7 +141,8 @@ public UnsignedLongTerms( otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + false ); } @@ -187,7 +188,14 @@ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) } @Override - protected UnsignedLongTerms create(String name, List buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) { + protected UnsignedLongTerms create( + String name, + List buckets, + BucketOrder reduceOrder, + long docCountError, + long otherDocCount, + boolean hasSliceLevelDocCountError + ) { return new UnsignedLongTerms( name, reduceOrder, @@ -275,7 +283,8 @@ static DoubleTerms convertUnsignedLongTermsToDouble(UnsignedLongTerms unsignedLo unsignedLongTerms.otherDocCount, newBuckets, unsignedLongTerms.docCountError, - unsignedLongTerms.bucketCountThresholds + unsignedLongTerms.bucketCountThresholds, + unsignedLongTerms.hasSliceLevelDocCountError ); } } diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java index cc43f4e5d79fb..02837da64dafd 100644 --- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java @@ -86,8 +86,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; -import static org.opensearch.search.aggregations.bucket.BucketUtils.suggestShardSideQueueSize; - /** * This class encapsulates the state needed to execute a search. It holds a reference to the * shards point in time snapshot (IndexReader / ContextIndexSearcher) and allows passing on @@ -410,11 +408,10 @@ public boolean shouldUseConcurrentSearch() { * Returns local bucket count thresholds based on concurrent segment search status */ public LocalBucketCountThresholds asLocalBucketCountThresholds(TermsAggregator.BucketCountThresholds bucketCountThresholds) { - if (shouldUseConcurrentSearch()) { - return new LocalBucketCountThresholds(0, suggestShardSideQueueSize(bucketCountThresholds.getShardSize())); - } else { - return new LocalBucketCountThresholds(bucketCountThresholds.getShardMinDocCount(), bucketCountThresholds.getShardSize()); - } + return new LocalBucketCountThresholds( + shouldUseConcurrentSearch() ? 0 : bucketCountThresholds.getShardMinDocCount(), + bucketCountThresholds.getShardSize() + ); } /** diff --git a/server/src/test/java/org/opensearch/search/aggregations/InternalAggregationsTests.java b/server/src/test/java/org/opensearch/search/aggregations/InternalAggregationsTests.java index 6fabd36a23b27..c68bf1ec3bb2e 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/InternalAggregationsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/InternalAggregationsTests.java @@ -84,7 +84,8 @@ public void testNonFinalReduceTopLevelPipelineAggs() { 10, Collections.emptyList(), 0, - new TermsAggregator.BucketCountThresholds(1, 0, 10, 25) + new TermsAggregator.BucketCountThresholds(1, 0, 10, 25), + false ); List aggs = singletonList(InternalAggregations.from(Collections.singletonList(terms))); InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(aggs, maxBucketReduceContext().forPartialReduction()); @@ -103,7 +104,8 @@ public void testFinalReduceTopLevelPipelineAggs() { 10, Collections.emptyList(), 0, - new TermsAggregator.BucketCountThresholds(1, 0, 10, 25) + new TermsAggregator.BucketCountThresholds(1, 0, 10, 25), + false ); InternalAggregations aggs = InternalAggregations.from(Collections.singletonList(terms)); diff --git a/server/src/test/java/org/opensearch/search/aggregations/InternalMultiBucketAggregationTests.java b/server/src/test/java/org/opensearch/search/aggregations/InternalMultiBucketAggregationTests.java index b7f4094da9990..cee9c4f1fb3e9 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/InternalMultiBucketAggregationTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/InternalMultiBucketAggregationTests.java @@ -176,7 +176,8 @@ public void testResolveToSpecificBucket() { 0, stringBuckets, 0, - new TermsAggregator.BucketCountThresholds(0, 0, 1, 1) + new TermsAggregator.BucketCountThresholds(0, 0, 1, 1), + false ); InternalAggregations internalAggregations = InternalAggregations.from(Collections.singletonList(termsAgg)); LongTerms.Bucket bucket = new LongTerms.Bucket(19, 1, internalAggregations, false, 0, DocValueFormat.RAW); @@ -214,7 +215,8 @@ public void testResolveToMissingSpecificBucket() { 0, stringBuckets, 0, - new TermsAggregator.BucketCountThresholds(0, 0, 1, 1) + new TermsAggregator.BucketCountThresholds(0, 0, 1, 1), + false ); InternalAggregations internalAggregations = InternalAggregations.from(Collections.singletonList(termsAgg)); LongTerms.Bucket bucket = new LongTerms.Bucket(19, 1, internalAggregations, false, 0, DocValueFormat.RAW); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/DoubleTermsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/DoubleTermsTests.java index 5fe9c1dee358d..efd7a41101a17 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/DoubleTermsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/DoubleTermsTests.java @@ -88,7 +88,8 @@ public class DoubleTermsTests extends InternalTermsTestCase { otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + false ); } @@ -170,7 +171,8 @@ protected Class implementationClass() { otherDocCount, buckets, docCountError, - new TermsAggregator.BucketCountThresholds(minDocCount, 0, requiredSize, shardSize) + new TermsAggregator.BucketCountThresholds(minDocCount, 0, requiredSize, shardSize), + false ); } else { String name = instance.getName(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/LongTermsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/LongTermsTests.java index 44fa9f5e79593..aaffa557619e3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/LongTermsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/LongTermsTests.java @@ -88,7 +88,8 @@ public class LongTermsTests extends InternalTermsTestCase { otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + false ); } @@ -170,7 +171,8 @@ protected Class implementationClass() { otherDocCount, buckets, docCountError, - new TermsAggregator.BucketCountThresholds(minDocCount, 0, requiredSize, shardSize) + new TermsAggregator.BucketCountThresholds(minDocCount, 0, requiredSize, shardSize), + false ); } else { String name = instance.getName(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/StringTermsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/StringTermsTests.java index deba96fd3ae19..fc435c55e89a9 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/StringTermsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/StringTermsTests.java @@ -147,7 +147,8 @@ protected Class implementationClass() { otherDocCount, buckets, docCountError, - new TermsAggregator.BucketCountThresholds(minDocCount, 0, requiredSize, shardSize) + new TermsAggregator.BucketCountThresholds(minDocCount, 0, requiredSize, shardSize), + false ); } else { String name = instance.getName(); @@ -238,7 +239,8 @@ private BytesRef[] generateRandomDict() { otherDocCount, buckets, docCountError, - bucketCountThresholds + bucketCountThresholds, + false ); } }