From 11be2f9d04c2b776fb89d532d191520982608c5e Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Thu, 7 Mar 2024 10:49:54 -0800 Subject: [PATCH] Change default NDV to 1024 Signed-off-by: Chen Dai --- .../adaptive/AdaptiveBloomFilter.java | 4 ++-- .../adaptive/AdaptiveBloomFilterTest.java | 18 +++++++++--------- .../BloomFilterSkippingStrategySuite.scala | 1 + 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/flint-core/src/main/java/org/opensearch/flint/core/field/bloomfilter/adaptive/AdaptiveBloomFilter.java b/flint-core/src/main/java/org/opensearch/flint/core/field/bloomfilter/adaptive/AdaptiveBloomFilter.java index e645d339c..9274eca64 100644 --- a/flint-core/src/main/java/org/opensearch/flint/core/field/bloomfilter/adaptive/AdaptiveBloomFilter.java +++ b/flint-core/src/main/java/org/opensearch/flint/core/field/bloomfilter/adaptive/AdaptiveBloomFilter.java @@ -34,7 +34,7 @@ public class AdaptiveBloomFilter implements BloomFilter { public AdaptiveBloomFilter(int numCandidate, double fpp) { this.candidates = new BloomFilterCandidate[numCandidate]; - int expectedNumItems = 128; + int expectedNumItems = 1024; for (int i = 0; i < candidates.length; i++) { candidates[i] = new BloomFilterCandidate( @@ -48,7 +48,7 @@ public AdaptiveBloomFilter(int numCandidate, double fpp) { this.total = total; this.candidates = new BloomFilterCandidate[candidates.length]; - int expectedNumItems = 128; + int expectedNumItems = 1024; for (int i = 0; i < candidates.length; i++) { this.candidates[i] = new BloomFilterCandidate(expectedNumItems, candidates[i]); expectedNumItems *= 2; diff --git a/flint-core/src/test/java/org/opensearch/flint/core/field/bloomfilter/adaptive/AdaptiveBloomFilterTest.java b/flint-core/src/test/java/org/opensearch/flint/core/field/bloomfilter/adaptive/AdaptiveBloomFilterTest.java index d7003fa15..8b7b30377 100644 --- a/flint-core/src/test/java/org/opensearch/flint/core/field/bloomfilter/adaptive/AdaptiveBloomFilterTest.java +++ b/flint-core/src/test/java/org/opensearch/flint/core/field/bloomfilter/adaptive/AdaptiveBloomFilterTest.java @@ -24,26 +24,26 @@ public class AdaptiveBloomFilterTest { @Test public void shouldChooseBestCandidateAdaptively() { - // Insert 50 items - for (int i = 0; i < 50; i++) { + // Insert 500 items + for (int i = 0; i < 500; i++) { bloomFilter.put(i); } BloomFilterCandidate candidate1 = bloomFilter.bestCandidate(); - assertEquals(128, candidate1.expectedNumItems); + assertEquals(1024, candidate1.expectedNumItems); - // Insert 100 (total 150) - for (int i = 50; i < 150; i++) { + // Insert 1000 (total 1500) + for (int i = 500; i < 1500; i++) { bloomFilter.put(i); } BloomFilterCandidate candidate2 = bloomFilter.bestCandidate(); - assertEquals(256, candidate2.expectedNumItems); + assertEquals(2048, candidate2.expectedNumItems); - // Insert 400 (total 550) - for (int i = 150; i < 550; i++) { + // Insert 4000 (total 5500) + for (int i = 1500; i < 5500; i++) { bloomFilter.put(i); } BloomFilterCandidate candidate3 = bloomFilter.bestCandidate(); - assertEquals(1024, candidate3.expectedNumItems); + assertEquals(8192, candidate3.expectedNumItems); } @Test diff --git a/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/skipping/bloomfilter/BloomFilterSkippingStrategySuite.scala b/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/skipping/bloomfilter/BloomFilterSkippingStrategySuite.scala index 2d7ace055..1a623e3e8 100644 --- a/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/skipping/bloomfilter/BloomFilterSkippingStrategySuite.scala +++ b/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/skipping/bloomfilter/BloomFilterSkippingStrategySuite.scala @@ -23,6 +23,7 @@ class BloomFilterSkippingStrategySuite test("parameters") { strategy.parameters shouldBe Map( BLOOM_FILTER_ADAPTIVE_KEY -> DEFAULT_BLOOM_FILTER_ADAPTIVE.toString, + ADAPTIVE_NUMBER_CANDIDATE_KEY -> DEFAULT_ADAPTIVE_NUMBER_CANDIDATE.toString, CLASSIC_BLOOM_FILTER_FPP_KEY -> DEFAULT_CLASSIC_BLOOM_FILTER_FPP.toString) } }