diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 23373fb84..834a2a201 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -13,7 +13,7 @@ sbt test ``` To run a specific unit test in SBT, use the testOnly command with the full path of the test class: ``` -sbt test:testOnly org.opensearch.flint.spark.ppl.PPLLogicalPlanTrendlineCommandTranslatorTestSuite +sbt "; project pplSparkIntegration; test:testOnly org.opensearch.flint.spark.ppl.PPLLogicalPlanTrendlineCommandTranslatorTestSuite" ``` @@ -30,7 +30,7 @@ If you get integration test failures with error message "Previous attempts to fi Running only a selected set of integration test suites is possible with the following command: ``` -sbt "project integtest" it:testOnly org.opensearch.flint.spark.ppl.FlintSparkPPLTrendlineITSuite +sbt "; project integtest; it:testOnly org.opensearch.flint.spark.ppl.FlintSparkPPLTrendlineITSuite" ``` This command runs only the specified test suite within the integtest submodule. diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index e232c3668..86970cefb 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -292,8 +292,6 @@ public enum BuiltinFunctionName { MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), WILDCARD_QUERY(FunctionName.of("wildcard_query")), - - NTH_VALUE(FunctionName.of("nth_value")), COALESCE(FunctionName.of("coalesce")); private FunctionName name; diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/TrendlineCatalystUtils.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/TrendlineCatalystUtils.java index 513561bfa..647f4542e 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/TrendlineCatalystUtils.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/TrendlineCatalystUtils.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl.utils; +import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction; import org.apache.spark.sql.catalyst.expressions.*; import org.opensearch.sql.ast.expression.*; import org.opensearch.sql.ast.expression.Literal; @@ -12,6 +13,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.ppl.CatalystExpressionVisitor; import org.opensearch.sql.ppl.CatalystPlanContext; +import scala.collection.mutable.Seq; import scala.Option; import scala.Tuple2; @@ -22,6 +24,8 @@ import java.util.stream.Collectors; import static org.opensearch.sql.ppl.utils.DataTypeTransformer.seq; +import static scala.Option.empty; +import static scala.collection.JavaConverters.asScalaBufferConverter; public interface TrendlineCatalystUtils { @@ -208,13 +212,14 @@ private static List getNthValueAggregations(CatalystExpressionVisito for (int i = 1; i <= dataPoints; i++) { // Get the offset parameter Expression offSetExpression = parseIntToExpression(visitor, context, i); - - // Composite the nth_value expression. - Function func = new Function(BuiltinFunctionName.NTH_VALUE.name(), - List.of(node.getDataField(), new Literal(i, DataType.INTEGER))); - - visitor.visitFunction(func, context); - Expression nthValueExp = context.popNamedParseExpressions().get(); + // Get the dataField in Expression + visitor.analyze(node.getDataField(), context); + Expression dataField = context.popNamedParseExpressions().get(); + // nth_value Expression + UnresolvedFunction nthValueExp = new UnresolvedFunction( + asScalaBufferConverter(List.of("nth_value")).asScala().seq(), + asScalaBufferConverter(List.of(dataField, offSetExpression)).asScala().seq(), + false, empty(), false); expressions.add(new Multiply( new WindowExpression(nthValueExp, windowDefinition), offSetExpression));