diff --git a/docs/ppl-lang/functions/ppl-datetime.md b/docs/ppl-lang/functions/ppl-datetime.md index e7b423d41..e479176a4 100644 --- a/docs/ppl-lang/functions/ppl-datetime.md +++ b/docs/ppl-lang/functions/ppl-datetime.md @@ -14,7 +14,7 @@ Argument type: DATE, LONG (DATE, LONG) -> DATE -Antonyms: `SUBDATE`_ +Antonyms: `SUBDATE` Example: @@ -795,7 +795,7 @@ Argument type: DATE/TIMESTAMP, LONG (DATE, LONG) -> DATE -Antonyms: `ADDDATE`_ +Antonyms: `ADDDATE` Example: @@ -982,3 +982,134 @@ Example: +----------------------------+ +### `DATE_ADD` + +**Description:** + +Usage: date_add(date, INTERVAL expr unit) adds the interval expr to date. + +Argument type: DATE, INTERVAL + +Return type: DATE + +Antonyms: `DATE_SUB` + +Example:: + + os> source=people | eval `'2020-08-26' + 1d` = DATE_ADD(DATE('2020-08-26'), INTERVAL 1 DAY) | fields `'2020-08-26' + 1d` + fetched rows / total rows = 1/1 + +---------------------+ + | '2020-08-26' + 1d | + |---------------------+ + | 2020-08-27 | + +---------------------+ + + +### `DATE_SUB` + +**Description:** + +Usage: date_sub(date, INTERVAL expr unit) subtracts the interval expr from date. + +Argument type: DATE, INTERVAL + +Return type: DATE + +Antonyms: `DATE_ADD` + +Example:: + + os> source=people | eval `'2008-01-02' - 31d` = DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY) | fields `'2008-01-02' - 31d` + fetched rows / total rows = 1/1 + +---------------------+ + | '2008-01-02' - 31d | + |---------------------+ + | 2007-12-02 | + +---------------------+ + + +### `TIMESTAMPADD` + +**Description:** + +Usage: Returns a TIMESTAMP value based on a passed in DATE/TIMESTAMP/STRING argument and an INTERVAL and INTEGER argument which determine the amount of time to be added. +If the third argument is a STRING, it must be formatted as a valid TIMESTAMP. +If the third argument is a DATE, it will be automatically converted to a TIMESTAMP. + +Argument type: INTERVAL, INTEGER, DATE/TIMESTAMP/STRING + +INTERVAL must be one of the following tokens: [SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR] + +Examples:: + + os> source=people | eval `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | eval `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` = TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | fields `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')`, `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` + fetched rows / total rows = 1/1 + +----------------------------------------------+--------------------------------------------------+ + | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | + |----------------------------------------------+--------------------------------------------------| + | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | + +----------------------------------------------+--------------------------------------------------+ + + +### `TIMESTAMPDIFF` + +**Description:** + +Usage: TIMESTAMPDIFF(interval, start, end) returns the difference between the start and end date/times in interval units. +Arguments will be automatically converted to a ]TIMESTAMP when appropriate. +Any argument that is a STRING must be formatted as a valid TIMESTAMP. + +Argument type: INTERVAL, DATE/TIMESTAMP/STRING, DATE/TIMESTAMP/STRING + +INTERVAL must be one of the following tokens: [SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR] + +Examples:: + + os> source=people | eval `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | eval `TIMESTAMPDIFF(SECOND, timestamp('1997-01-01 00:00:23'), timestamp('1997-01-01 00:00:00'))` = TIMESTAMPDIFF(SECOND, timestamp('1997-01-01 00:00:23'), timestamp('1997-01-01 00:00:00')) | fields `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')`, `TIMESTAMPDIFF(SECOND, timestamp('1997-01-01 00:00:23'), timestamp('1997-01-01 00:00:00'))` + fetched rows / total rows = 1/1 + +-------------------------------------------------------------------+-------------------------------------------------------------------------------------------+ + | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, timestamp('1997-01-01 00:00:23'), timestamp('1997-01-01 00:00:00')) | + |-------------------------------------------------------------------+-------------------------------------------------------------------------------------------| + | 4 | -23 | + +-------------------------------------------------------------------+-------------------------------------------------------------------------------------------+ + + +### `UTC_TIMESTAMP` + +**Description:** + +Returns the current UTC timestamp as a value in 'YYYY-MM-DD hh:mm:ss'. + +Return type: TIMESTAMP + +Specification: UTC_TIMESTAMP() -> TIMESTAMP + +Example:: + + > source=people | eval `UTC_TIMESTAMP()` = UTC_TIMESTAMP() | fields `UTC_TIMESTAMP()` + fetched rows / total rows = 1/1 + +---------------------+ + | UTC_TIMESTAMP() | + |---------------------| + | 2022-10-03 17:54:28 | + +---------------------+ + + +### `CURRENT_TIMEZONE` + +**Description:** + +Returns the current local timezone. + +Return type: STRING + +Example:: + + > source=people | eval `CURRENT_TIMEZONE()` = CURRENT_TIMEZONE() | fields `CURRENT_TIMEZONE()` + fetched rows / total rows = 1/1 + +------------------------+ + | CURRENT_TIMEZONE() | + |------------------------| + | America/Chicago | + +------------------------+ + diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala index 71ed72814..8001a690d 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala @@ -218,6 +218,117 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite comparePlans(logicalPlan, expectedPlan, checkAnalysis = false) } + test("test DATE_ADD") { + val frame1 = sql(s""" + | source = $testTable | eval `'2020-08-26' + 2d` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 DAY) + | | fields `'2020-08-26' + 2d` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-08-28"))), frame1) + + val frame2 = sql(s""" + | source = $testTable | eval `'2020-08-26' - 2d` = DATE_ADD(DATE('2020-08-26'), INTERVAL -2 DAY) + | | fields `'2020-08-26' - 2d` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-08-24"))), frame2) + + val frame3 = sql(s""" + | source = $testTable | eval `'2020-08-26' + 2m` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 MONTH) + | | fields `'2020-08-26' + 2m` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-10-26"))), frame3) + + val frame4 = sql(s""" + | source = $testTable | eval `'2020-08-26' + 2y` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 YEAR) + | | fields `'2020-08-26' + 2y` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2022-08-26"))), frame4) + + val ex = intercept[AnalysisException](sql(s""" + | source = $testTable | eval `'2020-08-26 01:01:01' + 2h` = DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 2 HOUR) + | | fields `'2020-08-26 01:01:01' + 2h` | head 1 + | """.stripMargin)) + assert(ex.getMessage.contains("""Parameter 1 requires the "DATE" type""")) + } + + test("test DATE_SUB") { + val frame1 = sql(s""" + | source = $testTable | eval `'2020-08-26' - 2d` = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 DAY) + | | fields `'2020-08-26' - 2d` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-08-24"))), frame1) + + val frame2 = sql(s""" + | source = $testTable | eval `'2020-08-26' + 2d` = DATE_SUB(DATE('2020-08-26'), INTERVAL -2 DAY) + | | fields `'2020-08-26' + 2d` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-08-28"))), frame2) + + val frame3 = sql(s""" + | source = $testTable | eval `'2020-08-26' - 2m` = DATE_SUB(DATE('2020-08-26'), INTERVAL 12 MONTH) + | | fields `'2020-08-26' - 2m` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2019-08-26"))), frame3) + + val frame4 = sql(s""" + | source = $testTable | eval `'2020-08-26' - 2y` = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 YEAR) + | | fields `'2020-08-26' - 2y` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2018-08-26"))), frame4) + + val ex = intercept[AnalysisException](sql(s""" + | source = $testTable | eval `'2020-08-26 01:01:01' - 2h` = DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 2 HOUR) + | | fields `'2020-08-26 01:01:01' - 2h` | head 1 + | """.stripMargin)) + assert(ex.getMessage.contains("""Parameter 1 requires the "DATE" type""")) + } + + test("test TIMESTAMPADD") { + val frame = sql(s""" + | source = $testTable + | | eval `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') + | | eval `TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00'))` = TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00')) + | | eval `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` = TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') + | | fields `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')`, `TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00'))`, `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` + | | head 1 + | """.stripMargin) + assertSameRows( + Seq( + Row( + Timestamp.valueOf("2000-01-18 00:00:00"), + Timestamp.valueOf("2000-01-18 00:00:00"), + Timestamp.valueOf("1999-10-01 00:00:00"))), + frame) + } + + test("test TIMESTAMPDIFF") { + val frame = sql(s""" + | source = $testTable + | | eval `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') + | | eval `TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00'))` = TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00')) + | | fields `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')`, `TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00'))` + | | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(4, -23)), frame) + } + + test("test CURRENT_TIMEZONE") { + val frame = sql(s""" + | source = $testTable + | | eval `CURRENT_TIMEZONE` = CURRENT_TIMEZONE() + | | fields `CURRENT_TIMEZONE` + | """.stripMargin) + assert(frame.collect().length > 0) + } + + test("test UTC_TIMESTAMP") { + val frame = sql(s""" + | source = $testTable + | | eval `UTC_TIMESTAMP` = UTC_TIMESTAMP() + | | fields `UTC_TIMESTAMP` + | """.stripMargin) + assert(frame.collect().length > 0) + } + test("test hour, minute, second, HOUR_OF_DAY, MINUTE_OF_HOUR") { val frame = sql(s""" | source = $testTable @@ -284,24 +395,6 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assert(ex.getMessage.contains("ADDTIME is not a builtin function of PPL")) } - test("test DATE_ADD is not supported") { - val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable - | | eval `DATE_ADD` = DATE_ADD() - | | fields DATE_ADD | head 1 - | """.stripMargin)) - assert(ex.getMessage.contains("DATE_ADD is not a builtin function of PPL")) - } - - test("test DATE_SUB is not supported") { - val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable - | | eval `DATE_SUB` = DATE_SUB() - | | fields DATE_SUB | head 1 - | """.stripMargin)) - assert(ex.getMessage.contains("DATE_SUB is not a builtin function of PPL")) - } - test("test DATETIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" | source = $testTable @@ -445,22 +538,6 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assert(ex.getMessage.contains("TIMEDIFF is not a builtin function of PPL")) } - test("test TIMESTAMPADD is not supported") { - intercept[Exception](sql(s""" - | source = $testTable - | | eval `TIMESTAMPADD` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') - | | fields TIMESTAMPADD | head 1 - | """.stripMargin)) - } - - test("test TIMESTAMPDIFF is not supported") { - intercept[Exception](sql(s""" - | source = $testTable - | | eval `TIMESTAMPDIFF_1` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') - | | fields TIMESTAMPDIFF_1 | head 1 - | """.stripMargin)) - } - test("test TO_DAYS is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" | source = $testTable @@ -497,15 +574,6 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assert(ex.getMessage.contains("UTC_TIME is not a builtin function of PPL")) } - test("test UTC_TIMESTAMP is not supported") { - val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable - | | eval `UTC_TIMESTAMP` = UTC_TIMESTAMP() - | | fields UTC_TIMESTAMP | head 1 - | """.stripMargin)) - assert(ex.getMessage.contains("UTC_TIMESTAMP is not a builtin function of PPL")) - } - test("test YEARWEEK is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" | source = $testTable diff --git a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 index a00e161c7..bf6989b7c 100644 --- a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -295,6 +295,7 @@ CURDATE: 'CURDATE'; CURRENT_DATE: 'CURRENT_DATE'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; +CURRENT_TIMEZONE: 'CURRENT_TIMEZONE'; CURTIME: 'CURTIME'; DATE: 'DATE'; DATEDIFF: 'DATEDIFF'; diff --git a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 index 4164843ef..aaf807a7b 100644 --- a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 @@ -421,6 +421,7 @@ valueExpression | primaryExpression # valueExpressionDefault | positionFunction # positionFunctionCall | caseFunction # caseExpr + | timestampFunction # timestampFunctionCall | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr ; @@ -677,6 +678,7 @@ dateTimeFunctionName | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP + | CURRENT_TIMEZONE | CURTIME | DATE | DATEDIFF @@ -893,6 +895,7 @@ literalValue | decimalLiteral | booleanLiteral | datetimeLiteral //#datetime + | intervalLiteral ; intervalLiteral diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Interval.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Interval.java index fc09ec2f5..bf00b2106 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Interval.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Interval.java @@ -23,11 +23,6 @@ public class Interval extends UnresolvedExpression { private final UnresolvedExpression value; private final IntervalUnit unit; - public Interval(UnresolvedExpression value, String unit) { - this.value = value; - this.unit = IntervalUnit.of(unit); - } - @Override public List getChild() { return Collections.singletonList(value); diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java index a7e983473..6e1e0712c 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java @@ -17,6 +17,7 @@ public enum IntervalUnit { UNKNOWN, MICROSECOND, + MILLISECOND, SECOND, MINUTE, HOUR, diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 9e1a9a743..d81dc7ce4 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -64,9 +64,9 @@ public enum BuiltinFunctionName { DATE(FunctionName.of("date")), DATEDIFF(FunctionName.of("datediff")), // DATETIME(FunctionName.of("datetime")), -// DATE_ADD(FunctionName.of("date_add")), + DATE_ADD(FunctionName.of("date_add")), DATE_FORMAT(FunctionName.of("date_format")), -// DATE_SUB(FunctionName.of("date_sub")), + DATE_SUB(FunctionName.of("date_sub")), DAY(FunctionName.of("day")), // DAYNAME(FunctionName.of("dayname")), DAYOFMONTH(FunctionName.of("dayofmonth")), @@ -105,14 +105,15 @@ public enum BuiltinFunctionName { // TIMEDIFF(FunctionName.of("timediff")), // TIME_TO_SEC(FunctionName.of("time_to_sec")), TIMESTAMP(FunctionName.of("timestamp")), -// TIMESTAMPADD(FunctionName.of("timestampadd")), -// TIMESTAMPDIFF(FunctionName.of("timestampdiff")), + TIMESTAMPADD(FunctionName.of("timestampadd")), + TIMESTAMPDIFF(FunctionName.of("timestampdiff")), // TIME_FORMAT(FunctionName.of("time_format")), // TO_DAYS(FunctionName.of("to_days")), // TO_SECONDS(FunctionName.of("to_seconds")), // UTC_DATE(FunctionName.of("utc_date")), // UTC_TIME(FunctionName.of("utc_time")), -// UTC_TIMESTAMP(FunctionName.of("utc_timestamp")), + UTC_TIMESTAMP(FunctionName.of("utc_timestamp")), + CURRENT_TIMEZONE(FunctionName.of("current_timezone")), UNIX_TIMESTAMP(FunctionName.of("unix_timestamp")), WEEK(FunctionName.of("week")), WEEKDAY(FunctionName.of("weekday")), diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/CatalystQueryPlanVisitor.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/CatalystQueryPlanVisitor.java index 87010f231..90df01e66 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/CatalystQueryPlanVisitor.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/CatalystQueryPlanVisitor.java @@ -20,6 +20,7 @@ import org.apache.spark.sql.catalyst.expressions.InSubquery$; import org.apache.spark.sql.catalyst.expressions.LessThanOrEqual; import org.apache.spark.sql.catalyst.expressions.ListQuery$; +import org.apache.spark.sql.catalyst.expressions.MakeInterval$; import org.apache.spark.sql.catalyst.expressions.NamedExpression; import org.apache.spark.sql.catalyst.expressions.Predicate; import org.apache.spark.sql.catalyst.expressions.ScalarSubquery$; @@ -114,6 +115,7 @@ import static java.util.List.of; import static org.opensearch.sql.expression.function.BuiltinFunctionName.EQUAL; import static org.opensearch.sql.ppl.CatalystPlanContext.findRelation; +import static org.opensearch.sql.ppl.utils.BuiltinFunctionTranslator.createIntervalArgs; import static org.opensearch.sql.ppl.utils.DataTypeTransformer.seq; import static org.opensearch.sql.ppl.utils.DataTypeTransformer.translate; import static org.opensearch.sql.ppl.utils.DedupeTransformer.retainMultipleDuplicateEvents; @@ -765,7 +767,13 @@ public Expression visitFillNull(FillNull fillNull, CatalystPlanContext context) @Override public Expression visitInterval(Interval node, CatalystPlanContext context) { - throw new IllegalStateException("Not Supported operation : Interval"); + node.getValue().accept(this, context); + Expression value = context.getNamedParseExpressions().pop(); + Expression[] intervalArgs = createIntervalArgs(node.getUnit(), value); + Expression interval = MakeInterval$.MODULE$.apply( + intervalArgs[0], intervalArgs[1], intervalArgs[2], intervalArgs[3], + intervalArgs[4], intervalArgs[5], intervalArgs[6], true); + return context.getNamedParseExpressions().push(interval); } @Override diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 0c7f6a9d4..6eb72c91e 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -422,12 +422,28 @@ public UnresolvedExpression visitInExpr(OpenSearchPPLParser.InExprContext ctx) { return ctx.NOT() != null ? new Not(expr) : expr; } - @Override public UnresolvedExpression visitCidrMatchFunctionCall(OpenSearchPPLParser.CidrMatchFunctionCallContext ctx) { return new Cidr(visit(ctx.ipAddress), visit(ctx.cidrBlock)); } + @Override + public UnresolvedExpression visitTimestampFunctionCall( + OpenSearchPPLParser.TimestampFunctionCallContext ctx) { + return new Function( + ctx.timestampFunction().timestampFunctionName().getText(), timestampFunctionArguments(ctx)); + } + + private List timestampFunctionArguments( + OpenSearchPPLParser.TimestampFunctionCallContext ctx) { + List args = + Arrays.asList( + new Literal(ctx.timestampFunction().simpleDateTimePart().getText(), DataType.STRING), + visitFunctionArg(ctx.timestampFunction().firstArg), + visitFunctionArg(ctx.timestampFunction().secondArg)); + return args; + } + private QualifiedName visitIdentifiers(List ctx) { return new QualifiedName( ctx.stream() diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java index 8982fe859..d954b04b9 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java @@ -8,10 +8,20 @@ import com.google.common.collect.ImmutableMap; import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction; import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction$; +import org.apache.spark.sql.catalyst.expressions.CurrentTimeZone$; +import org.apache.spark.sql.catalyst.expressions.CurrentTimestamp$; +import org.apache.spark.sql.catalyst.expressions.DateAddInterval$; import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.catalyst.expressions.Literal$; +import org.apache.spark.sql.catalyst.expressions.TimestampAdd$; +import org.apache.spark.sql.catalyst.expressions.TimestampDiff$; +import org.apache.spark.sql.catalyst.expressions.ToUTCTimestamp$; +import org.apache.spark.sql.catalyst.expressions.UnaryMinus$; +import org.opensearch.sql.ast.expression.IntervalUnit; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import scala.Option; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -19,6 +29,8 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADD; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDDATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATEDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_SUB; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_MONTH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.COALESCE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON; @@ -44,7 +56,10 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND_OF_MINUTE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBDATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.SYSDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIMESTAMP; import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK; import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK_OF_YEAR; import static org.opensearch.sql.ppl.utils.DataTypeTransformer.seq; @@ -95,8 +110,8 @@ public interface BuiltinFunctionTranslator { /** * The name mapping between PPL builtin functions to Spark builtin functions. */ - static final Map, UnresolvedFunction>> PPL_TO_SPARK_FUNC_MAPPING - = ImmutableMap., UnresolvedFunction>>builder() + static final Map, Expression>> PPL_TO_SPARK_FUNC_MAPPING + = ImmutableMap., Expression>>builder() // json functions .put( JSON_ARRAY, @@ -139,6 +154,31 @@ public interface BuiltinFunctionTranslator { seq(UnresolvedFunction$.MODULE$.apply("get_json_object", seq(args.get(0), Literal$.MODULE$.apply("$")), false)), false); }) + .put( + DATE_ADD, + args -> { + return DateAddInterval$.MODULE$.apply(args.get(0), args.get(1), Option.empty(), false); + }) + .put( + DATE_SUB, + args -> { + return DateAddInterval$.MODULE$.apply(args.get(0), UnaryMinus$.MODULE$.apply(args.get(1), true), Option.empty(), true); + }) + .put( + TIMESTAMPADD, + args -> { + return TimestampAdd$.MODULE$.apply(args.get(0).toString(), args.get(1), args.get(2), Option.empty()); + }) + .put( + TIMESTAMPDIFF, + args -> { + return TimestampDiff$.MODULE$.apply(args.get(0).toString(), args.get(1), args.get(2), Option.empty()); + }) + .put( + UTC_TIMESTAMP, + args -> { + return ToUTCTimestamp$.MODULE$.apply(CurrentTimestamp$.MODULE$.apply(), CurrentTimeZone$.MODULE$.apply()); + }) .build(); static Expression builtinFunction(org.opensearch.sql.ast.expression.Function function, List args) { @@ -153,7 +193,7 @@ static Expression builtinFunction(org.opensearch.sql.ast.expression.Function fun // there is a Spark builtin function mapping with the PPL builtin function return new UnresolvedFunction(seq(name), seq(args), false, empty(),false); } - Function, UnresolvedFunction> alternative = PPL_TO_SPARK_FUNC_MAPPING.get(builtin); + Function, Expression> alternative = PPL_TO_SPARK_FUNC_MAPPING.get(builtin); if (alternative != null) { return alternative.apply(args); } @@ -161,4 +201,21 @@ static Expression builtinFunction(org.opensearch.sql.ast.expression.Function fun return new UnresolvedFunction(seq(name), seq(args), false, empty(),false); } } + + static Expression[] createIntervalArgs(IntervalUnit unit, Expression value) { + Expression[] args = new Expression[7]; + Arrays.fill(args, Literal$.MODULE$.apply(0)); + switch (unit) { + case YEAR: args[0] = value; break; + case MONTH: args[1] = value; break; + case WEEK: args[2] = value; break; + case DAY: args[3] = value; break; + case HOUR: args[4] = value; break; + case MINUTE: args[5] = value; break; + case SECOND: args[6] = value; break; + default: + throw new IllegalArgumentException("Unsupported Interval unit: " + unit); + } + return args; + } } diff --git a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanDateTimeFunctionsTranslatorTestSuite.scala b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanDateTimeFunctionsTranslatorTestSuite.scala new file mode 100644 index 000000000..308b038bb --- /dev/null +++ b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanDateTimeFunctionsTranslatorTestSuite.scala @@ -0,0 +1,231 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.flint.spark.ppl + +import org.opensearch.flint.spark.ppl.PlaneUtils.plan +import org.opensearch.sql.ppl.{CatalystPlanContext, CatalystQueryPlanVisitor} +import org.scalatest.matchers.should.Matchers + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.analysis.{UnresolvedFunction, UnresolvedRelation, UnresolvedStar} +import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentTimestamp, CurrentTimeZone, DateAddInterval, Literal, MakeInterval, NamedExpression, TimestampAdd, TimestampDiff, ToUTCTimestamp, UnaryMinus} +import org.apache.spark.sql.catalyst.plans.PlanTest +import org.apache.spark.sql.catalyst.plans.logical.Project + +class PPLLogicalPlanDateTimeFunctionsTranslatorTestSuite + extends SparkFunSuite + with PlanTest + with LogicalPlanTestUtils + with Matchers { + + private val planTransformer = new CatalystQueryPlanVisitor() + private val pplParser = new PPLSyntaxParser() + + test("test DATE_ADD") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan(pplParser, "source=t | eval a = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 DAY)"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + DateAddInterval( + UnresolvedFunction("date", Seq(Literal("2020-08-26")), isDistinct = false), + MakeInterval( + Literal(0), + Literal(0), + Literal(0), + Literal(2), + Literal(0), + Literal(0), + Literal(0), + failOnError = true)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test DATE_ADD for year") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan(pplParser, "source=t | eval a = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 YEAR)"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + DateAddInterval( + UnresolvedFunction("date", Seq(Literal("2020-08-26")), isDistinct = false), + MakeInterval( + Literal(2), + Literal(0), + Literal(0), + Literal(0), + Literal(0), + Literal(0), + Literal(0), + failOnError = true)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test DATE_SUB") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan(pplParser, "source=t | eval a = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 DAY)"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + DateAddInterval( + UnresolvedFunction("date", Seq(Literal("2020-08-26")), isDistinct = false), + UnaryMinus( + MakeInterval( + Literal(0), + Literal(0), + Literal(0), + Literal(2), + Literal(0), + Literal(0), + Literal(0), + failOnError = true), + failOnError = true)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + assert(compareByString(expectedPlan) === compareByString(logPlan)) + } + + test("test TIMESTAMPADD") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan(pplParser, "source=t | eval a = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias(TimestampAdd("DAY", Literal(17), Literal("2000-01-01 00:00:00")), "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test TIMESTAMPADD with timestamp") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan( + pplParser, + "source=t | eval a = TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00'))"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + TimestampAdd( + "DAY", + Literal(17), + UnresolvedFunction( + "timestamp", + Seq(Literal("2000-01-01 00:00:00")), + isDistinct = false)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test TIMESTAMPDIFF") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan( + pplParser, + "source=t | eval a = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + TimestampDiff("YEAR", Literal("1997-01-01 00:00:00"), Literal("2001-03-06 00:00:00")), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test TIMESTAMPDIFF with timestamp") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan( + pplParser, + "source=t | eval a = TIMESTAMPDIFF(YEAR, TIMESTAMP('1997-01-01 00:00:00'), TIMESTAMP('2001-03-06 00:00:00'))"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + TimestampDiff( + "YEAR", + UnresolvedFunction( + "timestamp", + Seq(Literal("1997-01-01 00:00:00")), + isDistinct = false), + UnresolvedFunction( + "timestamp", + Seq(Literal("2001-03-06 00:00:00")), + isDistinct = false)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test UTC_TIMESTAMP") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit(plan(pplParser, "source=t | eval a = UTC_TIMESTAMP()"), context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias(ToUTCTimestamp(CurrentTimestamp(), CurrentTimeZone()), "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test CURRENT_TIMEZONE") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit(plan(pplParser, "source=t | eval a = CURRENT_TIMEZONE()"), context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias(UnresolvedFunction("current_timezone", Seq.empty, isDistinct = false), "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } +}