2.x uses t-digest 3.2

Signed-off-by: Lantao Jin <[email protected]>
opensearch-project · Jun 6, 2024 · 4b46f66 · 4b46f66
1 parent 3d784c2
commit 4b46f66
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 35 deletions.
diff --git a/core/build.gradle b/core/build.gradle
@@ -44,7 +44,7 @@ dependencies {
     api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
     api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
     api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
-    api group: 'com.tdunning', name: 't-digest', version: '3.3'
+    api group: 'com.tdunning', name: 't-digest', version: '3.2'
     api project(':common')
 
     testImplementation('org.junit.jupiter:junit-jupiter:5.9.3')

diff --git a/...c/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java b/...c/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java
@@ -46,13 +46,13 @@ public class PercentileApproxAggregatorTest extends AggregationTest {
   public void test_percentile_field_expression() {
     ExprValue result =
         aggregation(DSL.percentile(DSL.ref("integer_value", INTEGER), DSL.literal(50)), tuples);
-    assertEquals(3.0, result.value());
+    assertEquals(2.5, result.value());
     result = aggregation(DSL.percentile(DSL.ref("long_value", LONG), DSL.literal(50)), tuples);
-    assertEquals(3.0, result.value());
+    assertEquals(2.5, result.value());
     result = aggregation(DSL.percentile(DSL.ref("double_value", DOUBLE), DSL.literal(50)), tuples);
-    assertEquals(3.0, result.value());
+    assertEquals(2.5, result.value());
     result = aggregation(DSL.percentile(DSL.ref("float_value", FLOAT), DSL.literal(50)), tuples);
-    assertEquals(3.0, result.value());
+    assertEquals(2.5, result.value());
   }
 
   @Test
@@ -102,22 +102,23 @@ public void test_percentile_with_negative() {
             longValue(-50000L),
             longValue(40000L),
             longValue(50000L));
-    assertEquals(40000.0, result.value());
+    assertEquals(-5000.0, result.value());
     ExprValue[] results =
         percentiles(longValue(-100000L), longValue(-50000L), longValue(40000L), longValue(50000L));
     assertPercentileValues(
-        results, -100000.0, // p=1.0
+        results,
+        -100000.0, // p=1.0
         -100000.0, // p=5.0
         -100000.0, // p=10.0
-        -100000.0, // p=20.0
-        -50000.0, // p=25.0
-        -50000.0, // p=30.0
-        -50000.0, // p=40.0
-        40000.0, // p=50.0
-        40000.0, // p=60.0
-        40000.0, // p=70.0
-        50000.0, // p=75.0
-        50000.0, // p=80.0
+        -85000.0, // p=20.0
+        -75000.0, // p=25.0
+        -65000.0, // p=30.0
+        -40999.999999999985, // p=40.0
+        -5000.0, // p=50.0
+        30999.999999999996, // p=60.0
+        43000.0, // p=70.0
+        45000.0, // p=75.0
+        47000.0, // p=80.0
         50000.0, // p=90.0
         50000.0, // p=95.0
         50000.0, // p=99.0
@@ -134,15 +135,15 @@ public void test_percentile_value() {
         results, 0.0, // p=1.0
         0.0, // p=5.0
         0.0, // p=10.0
-        1.0, // p=20.0
-        1.0, // p=25.0
+        0.5, // p=20.0
+        0.75, // p=25.0
         1.0, // p=30.0
-        2.0, // p=40.0
+        1.5, // p=40.0
         2.0, // p=50.0
-        3.0, // p=60.0
+        2.5, // p=60.0
         3.0, // p=70.0
-        3.0, // p=75.0
-        4.0, // p=80.0
+        3.25, // p=75.0
+        3.5, // p=80.0
         4.0, // p=90.0
         4.0, // p=95.0
         4.0, // p=99.0
@@ -209,7 +210,7 @@ public void test_arithmetic_expression() {
                     DSL.literal(ExprValueUtils.integerValue(10))),
                 DSL.literal(50)),
             tuples);
-    assertEquals(30.0, result.value());
+    assertEquals(25.0, result.value());
   }
 
   @Test
@@ -228,7 +229,7 @@ public void test_with_missing() {
         aggregation(
             DSL.percentile(DSL.ref("integer_value", INTEGER), DSL.literal(50)),
             tuples_with_null_and_missing);
-    assertEquals(2.0, result.value());
+    assertEquals(1.5, result.value());
   }
 
   @Test
@@ -237,7 +238,7 @@ public void test_with_null() {
         aggregation(
             DSL.percentile(DSL.ref("double_value", DOUBLE), DSL.literal(50)),
             tuples_with_null_and_missing);
-    assertEquals(4.0, result.value());
+    assertEquals(3.5, result.value());
   }
 
   @Test

diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java
@@ -205,7 +205,7 @@ public void testStatsPercentileWithNull() throws IOException {
             String.format(
                 "source=%s | stats percentile(balance, 50)", TEST_INDEX_BANK_WITH_NULL_VALUES));
     verifySchema(response, schema("percentile(balance, 50)", null, "long"));
-    verifyDataRows(response, rows(39225));
+    verifyDataRows(response, rows(36031));
   }
 
   @Test
@@ -254,6 +254,6 @@ public void testStatsPercentileBySpan() throws IOException {
                 "source=%s | stats percentile(balance, 50) as p50 by span(age, 10) as age_bucket",
                 TEST_INDEX_BANK));
     verifySchema(response, schema("p50", null, "long"), schema("age_bucket", null, "integer"));
-    verifyDataRows(response, rows(32838, 20), rows(39225, 30));
+    verifyDataRows(response, rows(32838, 20), rows(27821, 30));
   }
 }
diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java
@@ -719,7 +719,7 @@ public void testFilteredPercentilePushDown() throws IOException {
             "SELECT percentile(balance, 50) FILTER(WHERE balance > 40000) FROM " + TEST_INDEX_BANK);
     verifySchema(
         response, schema("percentile(balance, 50) FILTER(WHERE balance > 40000)", null, "long"));
-    verifyDataRows(response, rows(48086));
+    verifyDataRows(response, rows(44313));
   }
 
   @Test
@@ -736,7 +736,7 @@ public void testPercentileGroupByPushDown() throws IOException {
         rows(39225, 32),
         rows(4180, 33),
         rows(48086, 34),
-        rows(16418, 36),
+        rows(11052, 36),
         rows(40540, 39));
   }
 

diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java
@@ -156,11 +156,11 @@ public void testPercentileOver() {
     verifyDataRowsInOrder(
         response,
         rows("Adams", 4180),
-        rows("Ayala", 40540),
+        rows("Ayala", 22360),
         rows("Bates", 32838),
-        rows("Bond", 32838),
+        rows("Bond", 19262),
         rows("Duke Willmington", 32838),
-        rows("Mcpherson", 39225),
+        rows("Mcpherson", 36031.5),
         rows("Ratliff", 32838));
   }
 
@@ -176,11 +176,11 @@ public void testPercentilePartition() {
     verifyDataRowsInOrder(
         response,
         rows("Ayala", 40540),
-        rows("Bates", 40540),
+        rows("Bates", 36689),
         rows("Mcpherson", 40540),
         rows("Adams", 4180),
-        rows("Bond", 5686),
+        rows("Bond", 4933),
         rows("Duke Willmington", 5686),
-        rows("Ratliff", 16418));
+        rows("Ratliff", 11052));
   }
 }