Refactor datetime functions docs and IT (opensearch-project#787)

Signed-off-by: Lantao Jin <[email protected]>

integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintPPLSuite.scala

@@ -8,6 +8,7 @@ package org.opensearch.flint.spark.ppl
 import org.opensearch.flint.spark.{FlintPPLSparkExtensions, FlintSparkExtensions, FlintSparkSuite}
 
 import org.apache.spark.SparkConf
+import org.apache.spark.sql.{DataFrame, QueryTest, Row}
 import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
 import org.apache.spark.sql.flint.config.FlintSparkConf.OPTIMIZER_RULE_ENABLED
@@ -24,4 +25,15 @@ trait FlintPPLSuite extends FlintSparkSuite {
       .set(OPTIMIZER_RULE_ENABLED.key, "false")
     conf
   }
+
+  def assertSameRows(expected: Seq[Row], df: DataFrame): Unit = {
+    QueryTest.sameRows(expected, df.collect().toSeq).foreach { results =>
+      fail(s"""
+           |Results do not match for query:
+           |${df.queryExecution}
+           |== Results ==
+           |$results
+         """.stripMargin)
+    }
+  }
 }

integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltinFunctionITSuite.scala

@@ -605,31 +605,6 @@ class FlintSparkPPLBuiltinFunctionITSuite
     comparePlans(logicalPlan, expectedPlan, checkAnalysis = false)
   }
 
-  test("test time functions - from_unixtime and unix_timestamp") {
-    val frame = sql(s"""
-       | source = $testTable |where unix_timestamp(from_unixtime(1700000001)) > 1700000000 | fields name, age
-       | """.stripMargin)
-
-    val results: Array[Row] = frame.collect()
-    val expectedResults: Array[Row] =
-      Array(Row("Jake", 70), Row("Hello", 30), Row("John", 25), Row("Jane", 20))
-    implicit val rowOrdering: Ordering[Row] = Ordering.by[Row, String](_.getAs[String](0))
-    assert(results.sorted.sameElements(expectedResults.sorted))
-
-    val logicalPlan: LogicalPlan = frame.queryExecution.logical
-    val table = UnresolvedRelation(Seq("spark_catalog", "default", "flint_ppl_test"))
-    val filterExpr = GreaterThan(
-      UnresolvedFunction(
-        "unix_timestamp",
-        seq(UnresolvedFunction("from_unixtime", seq(Literal(1700000001)), isDistinct = false)),
-        isDistinct = false),
-      Literal(1700000000))
-    val filterPlan = Filter(filterExpr, table)
-    val projectList = Seq(UnresolvedAttribute("name"), UnresolvedAttribute("age"))
-    val expectedPlan = Project(projectList, filterPlan)
-    comparePlans(logicalPlan, expectedPlan, checkAnalysis = false)
-  }
-
   test("test arithmetic operators (+ - * / %)") {
     val frame = sql(s"""
        | source = $testTable | where (sqrt(pow(age, 2)) + sqrt(pow(age, 2)) / 1 - sqrt(pow(age, 2)) * 1) % 25.0 = 0 | fields name, age

ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4

@@ -308,6 +308,7 @@ LAST_DAY:                           'LAST_DAY';
 LOCALTIME:                          'LOCALTIME';
 LOCALTIMESTAMP:                     'LOCALTIMESTAMP';
 MAKEDATE:                           'MAKEDATE';
+MAKE_DATE:                          'MAKE_DATE';
 MAKETIME:                           'MAKETIME';
 MONTHNAME:                          'MONTHNAME';
 NOW:                                'NOW';

ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4

@@ -655,6 +655,7 @@ dateTimeFunctionName
    | LOCALTIME
    | LOCALTIMESTAMP
    | MAKEDATE
+   | MAKE_DATE
    | MAKETIME
    | MICROSECOND
    | MINUTE

ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -54,73 +54,74 @@ public enum BuiltinFunctionName {
 
   /** Date and Time Functions. */
   ADDDATE(FunctionName.of("adddate")),
-  ADDTIME(FunctionName.of("addtime")),
-  CONVERT_TZ(FunctionName.of("convert_tz")),
+//  ADDTIME(FunctionName.of("addtime")),
+//  CONVERT_TZ(FunctionName.of("convert_tz")),
   DATE(FunctionName.of("date")),
   DATEDIFF(FunctionName.of("datediff")),
-  DATETIME(FunctionName.of("datetime")),
-  DATE_ADD(FunctionName.of("date_add")),
+//  DATETIME(FunctionName.of("datetime")),
+//  DATE_ADD(FunctionName.of("date_add")),
   DATE_FORMAT(FunctionName.of("date_format")),
-  DATE_SUB(FunctionName.of("date_sub")),
+//  DATE_SUB(FunctionName.of("date_sub")),
   DAY(FunctionName.of("day")),
-  DAYNAME(FunctionName.of("dayname")),
+//  DAYNAME(FunctionName.of("dayname")),
   DAYOFMONTH(FunctionName.of("dayofmonth")),
   DAY_OF_MONTH(FunctionName.of("day_of_month")),
   DAYOFWEEK(FunctionName.of("dayofweek")),
   DAYOFYEAR(FunctionName.of("dayofyear")),
   DAY_OF_WEEK(FunctionName.of("day_of_week")),
   DAY_OF_YEAR(FunctionName.of("day_of_year")),
   EXTRACT(FunctionName.of("extract")),
-  FROM_DAYS(FunctionName.of("from_days")),
+//  FROM_DAYS(FunctionName.of("from_days")),
   FROM_UNIXTIME(FunctionName.of("from_unixtime")),
-  GET_FORMAT(FunctionName.of("get_format")),
+//  GET_FORMAT(FunctionName.of("get_format")),
   HOUR(FunctionName.of("hour")),
   HOUR_OF_DAY(FunctionName.of("hour_of_day")),
   LAST_DAY(FunctionName.of("last_day")),
   MAKEDATE(FunctionName.of("makedate")),
-  MAKETIME(FunctionName.of("maketime")),
-  MICROSECOND(FunctionName.of("microsecond")),
+  MAKE_DATE(FunctionName.of("make_date")),
+//  MAKETIME(FunctionName.of("maketime")),
+//  MICROSECOND(FunctionName.of("microsecond")),
   MINUTE(FunctionName.of("minute")),
-  MINUTE_OF_DAY(FunctionName.of("minute_of_day")),
+//  MINUTE_OF_DAY(FunctionName.of("minute_of_day")),
   MINUTE_OF_HOUR(FunctionName.of("minute_of_hour")),
   MONTH(FunctionName.of("month")),
   MONTH_OF_YEAR(FunctionName.of("month_of_year")),
   MONTHNAME(FunctionName.of("monthname")),
-  PERIOD_ADD(FunctionName.of("period_add")),
-  PERIOD_DIFF(FunctionName.of("period_diff")),
+//  PERIOD_ADD(FunctionName.of("period_add")),
+//  PERIOD_DIFF(FunctionName.of("period_diff")),
   QUARTER(FunctionName.of("quarter")),
-  SEC_TO_TIME(FunctionName.of("sec_to_time")),
+//  SEC_TO_TIME(FunctionName.of("sec_to_time")),
   SECOND(FunctionName.of("second")),
   SECOND_OF_MINUTE(FunctionName.of("second_of_minute")),
-  STR_TO_DATE(FunctionName.of("str_to_date")),
+//  STR_TO_DATE(FunctionName.of("str_to_date")),
   SUBDATE(FunctionName.of("subdate")),
-  SUBTIME(FunctionName.of("subtime")),
-  TIME(FunctionName.of("time")),
-  TIMEDIFF(FunctionName.of("timediff")),
-  TIME_TO_SEC(FunctionName.of("time_to_sec")),
+//  SUBTIME(FunctionName.of("subtime")),
+//  TIME(FunctionName.of("time")),
+//  TIMEDIFF(FunctionName.of("timediff")),
+//  TIME_TO_SEC(FunctionName.of("time_to_sec")),
   TIMESTAMP(FunctionName.of("timestamp")),
-  TIMESTAMPADD(FunctionName.of("timestampadd")),
-  TIMESTAMPDIFF(FunctionName.of("timestampdiff")),
-  TIME_FORMAT(FunctionName.of("time_format")),
-  TO_DAYS(FunctionName.of("to_days")),
-  TO_SECONDS(FunctionName.of("to_seconds")),
-  UTC_DATE(FunctionName.of("utc_date")),
-  UTC_TIME(FunctionName.of("utc_time")),
-  UTC_TIMESTAMP(FunctionName.of("utc_timestamp")),
+//  TIMESTAMPADD(FunctionName.of("timestampadd")),
+//  TIMESTAMPDIFF(FunctionName.of("timestampdiff")),
+//  TIME_FORMAT(FunctionName.of("time_format")),
+//  TO_DAYS(FunctionName.of("to_days")),
+//  TO_SECONDS(FunctionName.of("to_seconds")),
+//  UTC_DATE(FunctionName.of("utc_date")),
+//  UTC_TIME(FunctionName.of("utc_time")),
+//  UTC_TIMESTAMP(FunctionName.of("utc_timestamp")),
   UNIX_TIMESTAMP(FunctionName.of("unix_timestamp")),
   WEEK(FunctionName.of("week")),
   WEEKDAY(FunctionName.of("weekday")),
   WEEKOFYEAR(FunctionName.of("weekofyear")),
   WEEK_OF_YEAR(FunctionName.of("week_of_year")),
   YEAR(FunctionName.of("year")),
-  YEARWEEK(FunctionName.of("yearweek")),
+//  YEARWEEK(FunctionName.of("yearweek")),
 
   // `now`-like functions
   NOW(FunctionName.of("now")),
   CURDATE(FunctionName.of("curdate")),
   CURRENT_DATE(FunctionName.of("current_date")),
-  CURTIME(FunctionName.of("curtime")),
-  CURRENT_TIME(FunctionName.of("current_time")),
+//  CURTIME(FunctionName.of("curtime")),
+//  CURRENT_TIME(FunctionName.of("current_time")),
   LOCALTIME(FunctionName.of("localtime")),
   CURRENT_TIMESTAMP(FunctionName.of("current_timestamp")),
   LOCALTIMESTAMP(FunctionName.of("localtimestamp")),

ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java

@@ -33,6 +33,7 @@
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND_OF_MINUTE;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBDATE;
+import static org.opensearch.sql.expression.function.BuiltinFunctionName.SYSDATE;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRIM;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK_OF_YEAR;
@@ -66,7 +67,8 @@ public interface BuiltinFunctionTranslator {
             .put(ADDDATE, "date_add") // only maps adddate(date, days)
             .put(DATEDIFF, "datediff")
             .put(LOCALTIME, "localtimestamp")
-            //condition functions
+            .put(SYSDATE, "now")
+            // condition functions
             .put(IS_NULL, "isnull")
             .put(IS_NOT_NULL, "isnotnull")
             .put(BuiltinFunctionName.ISPRESENT, "isnotnull")