From 61c6bb846a680c14f931556b147bd7afd075bc29 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Fri, 8 Nov 2024 19:53:00 -0800 Subject: [PATCH] update license and scalafmt Signed-off-by: YANGDB --- .../ppl/FlintSparkPPLTopAndRareITSuite.scala | 41 ++++++++++++------- .../sql/ast/tree/CountedAggregation.java | 7 ++++ ...ggregationQueriesTranslatorTestSuite.scala | 16 +++++--- ...TopAndRareQueriesTranslatorTestSuite.scala | 7 +++- 4 files changed, 49 insertions(+), 22 deletions(-) diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLTopAndRareITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLTopAndRareITSuite.scala index dab6fd1aa..4a1633035 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLTopAndRareITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLTopAndRareITSuite.scala @@ -92,7 +92,7 @@ class FlintSparkPPLTopAndRareITSuite // Retrieve the results val results: Array[Row] = frame.collect() assert(results.length == 3) - + // Retrieve the logical plan val logicalPlan: LogicalPlan = frame.queryExecution.logical // Define the expected logical plan @@ -114,7 +114,10 @@ class FlintSparkPPLTopAndRareITSuite Seq( SortOrder( Alias( - UnresolvedFunction(Seq("APPROX_COUNT_DISTINCT"), Seq(addressField), isDistinct = false), + UnresolvedFunction( + Seq("APPROX_COUNT_DISTINCT"), + Seq(addressField), + isDistinct = false), "count_address")(), Ascending)), global = true, @@ -170,7 +173,7 @@ class FlintSparkPPLTopAndRareITSuite val expectedPlan = Project(projectList, sortedPlan) comparePlans(expectedPlan, logicalPlan, false) } - + test("create ppl rare 3 address by age field query test") { val frame = sql(s""" | source = $testTable| rare 3 address by age @@ -220,7 +223,7 @@ class FlintSparkPPLTopAndRareITSuite val expectedPlan = Project(Seq(UnresolvedStar(None)), planWithLimit) comparePlans(expectedPlan, logicalPlan, false) } - + test("create ppl rare 3 address by age field query test with approximation") { val frame = sql(s""" | source = $testTable| rare_approx 3 address by age @@ -230,7 +233,6 @@ class FlintSparkPPLTopAndRareITSuite val results: Array[Row] = frame.collect() assert(results.length == 3) - // Retrieve the logical plan val logicalPlan: LogicalPlan = frame.queryExecution.logical val addressField = UnresolvedAttribute("address") @@ -255,7 +257,10 @@ class FlintSparkPPLTopAndRareITSuite Seq( SortOrder( Alias( - UnresolvedFunction(Seq("APPROX_COUNT_DISTINCT"), Seq(addressField), isDistinct = false), + UnresolvedFunction( + Seq("APPROX_COUNT_DISTINCT"), + Seq(addressField), + isDistinct = false), "count_address")(), Ascending)), global = true, @@ -313,7 +318,7 @@ class FlintSparkPPLTopAndRareITSuite val expectedPlan = Project(projectList, sortedPlan) comparePlans(expectedPlan, logicalPlan, checkAnalysis = false) } - + test("create ppl top address field query test with approximation") { val frame = sql(s""" | source = $testTable| top_approx address @@ -323,7 +328,6 @@ class FlintSparkPPLTopAndRareITSuite val results: Array[Row] = frame.collect() assert(results.length == 3) - // Retrieve the logical plan val logicalPlan: LogicalPlan = frame.queryExecution.logical // Define the expected logical plan @@ -345,7 +349,10 @@ class FlintSparkPPLTopAndRareITSuite Seq( SortOrder( Alias( - UnresolvedFunction(Seq("APPROX_COUNT_DISTINCT"), Seq(addressField), isDistinct = false), + UnresolvedFunction( + Seq("APPROX_COUNT_DISTINCT"), + Seq(addressField), + isDistinct = false), "count_address")(), Descending)), global = true, @@ -400,7 +407,7 @@ class FlintSparkPPLTopAndRareITSuite val expectedPlan = Project(Seq(UnresolvedStar(None)), planWithLimit) comparePlans(expectedPlan, logicalPlan, checkAnalysis = false) } - + test("create ppl top 3 countries query test with approximation") { val frame = sql(s""" | source = $newTestTable| top_approx 3 country @@ -409,7 +416,7 @@ class FlintSparkPPLTopAndRareITSuite // Retrieve the results val results: Array[Row] = frame.collect() assert(results.length == 3) - + // Retrieve the logical plan val logicalPlan: LogicalPlan = frame.queryExecution.logical val countryField = UnresolvedAttribute("country") @@ -428,7 +435,10 @@ class FlintSparkPPLTopAndRareITSuite Seq( SortOrder( Alias( - UnresolvedFunction(Seq("APPROX_COUNT_DISTINCT"), Seq(countryField), isDistinct = false), + UnresolvedFunction( + Seq("APPROX_COUNT_DISTINCT"), + Seq(countryField), + isDistinct = false), "count_country")(), Descending)), global = true, @@ -491,7 +501,7 @@ class FlintSparkPPLTopAndRareITSuite comparePlans(expectedPlan, logicalPlan, checkAnalysis = false) } - + test("create ppl top 2 countries by occupation field query test with approximation") { val frame = sql(s""" | source = $newTestTable| top_approx 3 country by occupation @@ -522,7 +532,10 @@ class FlintSparkPPLTopAndRareITSuite Seq( SortOrder( Alias( - UnresolvedFunction(Seq("APPROX_COUNT_DISTINCT"), Seq(countryField), isDistinct = false), + UnresolvedFunction( + Seq("APPROX_COUNT_DISTINCT"), + Seq(countryField), + isDistinct = false), "count_country")(), Descending)), global = true, diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/tree/CountedAggregation.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/tree/CountedAggregation.java index c94779345..9a4aa5d7d 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/tree/CountedAggregation.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/tree/CountedAggregation.java @@ -1,9 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.sql.ast.tree; import org.opensearch.sql.ast.expression.Literal; import java.util.Optional; +/** + * marker interface for numeric based count aggregation (specific number of returned results) + */ public interface CountedAggregation { Optional getResults(); } diff --git a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanAggregationQueriesTranslatorTestSuite.scala b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanAggregationQueriesTranslatorTestSuite.scala index bda6bdf70..42cc7ed10 100644 --- a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanAggregationQueriesTranslatorTestSuite.scala +++ b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanAggregationQueriesTranslatorTestSuite.scala @@ -753,11 +753,13 @@ class PPLLogicalPlanAggregationQueriesTranslatorTestSuite comparePlans(expectedPlan, logPlan, false) } - + test("test approx distinct count product group by brand sorted") { val context = new CatalystPlanContext val logPlan = planTransformer.visit( - plan(pplParser, "source = table | stats distinct_count_approx(product) by brand | sort brand"), + plan( + pplParser, + "source = table | stats distinct_count_approx(product) by brand | sort brand"), context) val star = Seq(UnresolvedStar(None)) val brandField = UnresolvedAttribute("brand") @@ -828,8 +830,9 @@ class PPLLogicalPlanAggregationQueriesTranslatorTestSuite comparePlans(expectedPlan, logPlan, false) } - - test("test distinct count age by span of interval of 10 years query with sort using approximation ") { + + test( + "test distinct count age by span of interval of 10 years query with sort using approximation ") { val context = new CatalystPlanContext val logPlan = planTransformer.visit( plan( @@ -890,8 +893,9 @@ class PPLLogicalPlanAggregationQueriesTranslatorTestSuite // Compare the two plans comparePlans(expectedPlan, logPlan, false) } - - test("test distinct count status by week window and group by status with limit using approximation") { + + test( + "test distinct count status by week window and group by status with limit using approximation") { val context = new CatalystPlanContext val logPlan = planTransformer.visit( plan( diff --git a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanTopAndRareQueriesTranslatorTestSuite.scala b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanTopAndRareQueriesTranslatorTestSuite.scala index 97f8e2d53..106cba93a 100644 --- a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanTopAndRareQueriesTranslatorTestSuite.scala +++ b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanTopAndRareQueriesTranslatorTestSuite.scala @@ -58,7 +58,7 @@ class PPLLogicalPlanTopAndRareQueriesTranslatorTestSuite val expectedPlan = Project(projectList, sortedPlan) comparePlans(expectedPlan, logPlan, checkAnalysis = false) } - + test("test simple rare command with a single field approximation") { // if successful build ppl logical plan and translate to catalyst logical plan val context = new CatalystPlanContext @@ -83,7 +83,10 @@ class PPLLogicalPlanTopAndRareQueriesTranslatorTestSuite Seq( SortOrder( Alias( - UnresolvedFunction(Seq("APPROX_COUNT_DISTINCT"), Seq(addressField), isDistinct = false), + UnresolvedFunction( + Seq("APPROX_COUNT_DISTINCT"), + Seq(addressField), + isDistinct = false), "count_address")(), Ascending)), global = true,