diff --git a/docs/ppl-lang/functions/ppl-datetime.md b/docs/ppl-lang/functions/ppl-datetime.md index e7b423d41..e479176a4 100644 --- a/docs/ppl-lang/functions/ppl-datetime.md +++ b/docs/ppl-lang/functions/ppl-datetime.md @@ -14,7 +14,7 @@ Argument type: DATE, LONG (DATE, LONG) -> DATE -Antonyms: `SUBDATE`_ +Antonyms: `SUBDATE` Example: @@ -795,7 +795,7 @@ Argument type: DATE/TIMESTAMP, LONG (DATE, LONG) -> DATE -Antonyms: `ADDDATE`_ +Antonyms: `ADDDATE` Example: @@ -982,3 +982,134 @@ Example: +----------------------------+ +### `DATE_ADD` + +**Description:** + +Usage: date_add(date, INTERVAL expr unit) adds the interval expr to date. + +Argument type: DATE, INTERVAL + +Return type: DATE + +Antonyms: `DATE_SUB` + +Example:: + + os> source=people | eval `'2020-08-26' + 1d` = DATE_ADD(DATE('2020-08-26'), INTERVAL 1 DAY) | fields `'2020-08-26' + 1d` + fetched rows / total rows = 1/1 + +---------------------+ + | '2020-08-26' + 1d | + |---------------------+ + | 2020-08-27 | + +---------------------+ + + +### `DATE_SUB` + +**Description:** + +Usage: date_sub(date, INTERVAL expr unit) subtracts the interval expr from date. + +Argument type: DATE, INTERVAL + +Return type: DATE + +Antonyms: `DATE_ADD` + +Example:: + + os> source=people | eval `'2008-01-02' - 31d` = DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY) | fields `'2008-01-02' - 31d` + fetched rows / total rows = 1/1 + +---------------------+ + | '2008-01-02' - 31d | + |---------------------+ + | 2007-12-02 | + +---------------------+ + + +### `TIMESTAMPADD` + +**Description:** + +Usage: Returns a TIMESTAMP value based on a passed in DATE/TIMESTAMP/STRING argument and an INTERVAL and INTEGER argument which determine the amount of time to be added. +If the third argument is a STRING, it must be formatted as a valid TIMESTAMP. +If the third argument is a DATE, it will be automatically converted to a TIMESTAMP. + +Argument type: INTERVAL, INTEGER, DATE/TIMESTAMP/STRING + +INTERVAL must be one of the following tokens: [SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR] + +Examples:: + + os> source=people | eval `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | eval `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` = TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | fields `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')`, `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` + fetched rows / total rows = 1/1 + +----------------------------------------------+--------------------------------------------------+ + | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | + |----------------------------------------------+--------------------------------------------------| + | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | + +----------------------------------------------+--------------------------------------------------+ + + +### `TIMESTAMPDIFF` + +**Description:** + +Usage: TIMESTAMPDIFF(interval, start, end) returns the difference between the start and end date/times in interval units. +Arguments will be automatically converted to a ]TIMESTAMP when appropriate. +Any argument that is a STRING must be formatted as a valid TIMESTAMP. + +Argument type: INTERVAL, DATE/TIMESTAMP/STRING, DATE/TIMESTAMP/STRING + +INTERVAL must be one of the following tokens: [SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR] + +Examples:: + + os> source=people | eval `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | eval `TIMESTAMPDIFF(SECOND, timestamp('1997-01-01 00:00:23'), timestamp('1997-01-01 00:00:00'))` = TIMESTAMPDIFF(SECOND, timestamp('1997-01-01 00:00:23'), timestamp('1997-01-01 00:00:00')) | fields `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')`, `TIMESTAMPDIFF(SECOND, timestamp('1997-01-01 00:00:23'), timestamp('1997-01-01 00:00:00'))` + fetched rows / total rows = 1/1 + +-------------------------------------------------------------------+-------------------------------------------------------------------------------------------+ + | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, timestamp('1997-01-01 00:00:23'), timestamp('1997-01-01 00:00:00')) | + |-------------------------------------------------------------------+-------------------------------------------------------------------------------------------| + | 4 | -23 | + +-------------------------------------------------------------------+-------------------------------------------------------------------------------------------+ + + +### `UTC_TIMESTAMP` + +**Description:** + +Returns the current UTC timestamp as a value in 'YYYY-MM-DD hh:mm:ss'. + +Return type: TIMESTAMP + +Specification: UTC_TIMESTAMP() -> TIMESTAMP + +Example:: + + > source=people | eval `UTC_TIMESTAMP()` = UTC_TIMESTAMP() | fields `UTC_TIMESTAMP()` + fetched rows / total rows = 1/1 + +---------------------+ + | UTC_TIMESTAMP() | + |---------------------| + | 2022-10-03 17:54:28 | + +---------------------+ + + +### `CURRENT_TIMEZONE` + +**Description:** + +Returns the current local timezone. + +Return type: STRING + +Example:: + + > source=people | eval `CURRENT_TIMEZONE()` = CURRENT_TIMEZONE() | fields `CURRENT_TIMEZONE()` + fetched rows / total rows = 1/1 + +------------------------+ + | CURRENT_TIMEZONE() | + |------------------------| + | America/Chicago | + +------------------------+ + diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala index 71ed72814..8001a690d 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala @@ -218,6 +218,117 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite comparePlans(logicalPlan, expectedPlan, checkAnalysis = false) } + test("test DATE_ADD") { + val frame1 = sql(s""" + | source = $testTable | eval `'2020-08-26' + 2d` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 DAY) + | | fields `'2020-08-26' + 2d` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-08-28"))), frame1) + + val frame2 = sql(s""" + | source = $testTable | eval `'2020-08-26' - 2d` = DATE_ADD(DATE('2020-08-26'), INTERVAL -2 DAY) + | | fields `'2020-08-26' - 2d` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-08-24"))), frame2) + + val frame3 = sql(s""" + | source = $testTable | eval `'2020-08-26' + 2m` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 MONTH) + | | fields `'2020-08-26' + 2m` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-10-26"))), frame3) + + val frame4 = sql(s""" + | source = $testTable | eval `'2020-08-26' + 2y` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 YEAR) + | | fields `'2020-08-26' + 2y` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2022-08-26"))), frame4) + + val ex = intercept[AnalysisException](sql(s""" + | source = $testTable | eval `'2020-08-26 01:01:01' + 2h` = DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 2 HOUR) + | | fields `'2020-08-26 01:01:01' + 2h` | head 1 + | """.stripMargin)) + assert(ex.getMessage.contains("""Parameter 1 requires the "DATE" type""")) + } + + test("test DATE_SUB") { + val frame1 = sql(s""" + | source = $testTable | eval `'2020-08-26' - 2d` = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 DAY) + | | fields `'2020-08-26' - 2d` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-08-24"))), frame1) + + val frame2 = sql(s""" + | source = $testTable | eval `'2020-08-26' + 2d` = DATE_SUB(DATE('2020-08-26'), INTERVAL -2 DAY) + | | fields `'2020-08-26' + 2d` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2020-08-28"))), frame2) + + val frame3 = sql(s""" + | source = $testTable | eval `'2020-08-26' - 2m` = DATE_SUB(DATE('2020-08-26'), INTERVAL 12 MONTH) + | | fields `'2020-08-26' - 2m` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2019-08-26"))), frame3) + + val frame4 = sql(s""" + | source = $testTable | eval `'2020-08-26' - 2y` = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 YEAR) + | | fields `'2020-08-26' - 2y` | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(Date.valueOf("2018-08-26"))), frame4) + + val ex = intercept[AnalysisException](sql(s""" + | source = $testTable | eval `'2020-08-26 01:01:01' - 2h` = DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 2 HOUR) + | | fields `'2020-08-26 01:01:01' - 2h` | head 1 + | """.stripMargin)) + assert(ex.getMessage.contains("""Parameter 1 requires the "DATE" type""")) + } + + test("test TIMESTAMPADD") { + val frame = sql(s""" + | source = $testTable + | | eval `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') + | | eval `TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00'))` = TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00')) + | | eval `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` = TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') + | | fields `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')`, `TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00'))`, `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` + | | head 1 + | """.stripMargin) + assertSameRows( + Seq( + Row( + Timestamp.valueOf("2000-01-18 00:00:00"), + Timestamp.valueOf("2000-01-18 00:00:00"), + Timestamp.valueOf("1999-10-01 00:00:00"))), + frame) + } + + test("test TIMESTAMPDIFF") { + val frame = sql(s""" + | source = $testTable + | | eval `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') + | | eval `TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00'))` = TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00')) + | | fields `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')`, `TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00'))` + | | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(4, -23)), frame) + } + + test("test CURRENT_TIMEZONE") { + val frame = sql(s""" + | source = $testTable + | | eval `CURRENT_TIMEZONE` = CURRENT_TIMEZONE() + | | fields `CURRENT_TIMEZONE` + | """.stripMargin) + assert(frame.collect().length > 0) + } + + test("test UTC_TIMESTAMP") { + val frame = sql(s""" + | source = $testTable + | | eval `UTC_TIMESTAMP` = UTC_TIMESTAMP() + | | fields `UTC_TIMESTAMP` + | """.stripMargin) + assert(frame.collect().length > 0) + } + test("test hour, minute, second, HOUR_OF_DAY, MINUTE_OF_HOUR") { val frame = sql(s""" | source = $testTable @@ -284,24 +395,6 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assert(ex.getMessage.contains("ADDTIME is not a builtin function of PPL")) } - test("test DATE_ADD is not supported") { - val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable - | | eval `DATE_ADD` = DATE_ADD() - | | fields DATE_ADD | head 1 - | """.stripMargin)) - assert(ex.getMessage.contains("DATE_ADD is not a builtin function of PPL")) - } - - test("test DATE_SUB is not supported") { - val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable - | | eval `DATE_SUB` = DATE_SUB() - | | fields DATE_SUB | head 1 - | """.stripMargin)) - assert(ex.getMessage.contains("DATE_SUB is not a builtin function of PPL")) - } - test("test DATETIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" | source = $testTable @@ -445,22 +538,6 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assert(ex.getMessage.contains("TIMEDIFF is not a builtin function of PPL")) } - test("test TIMESTAMPADD is not supported") { - intercept[Exception](sql(s""" - | source = $testTable - | | eval `TIMESTAMPADD` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') - | | fields TIMESTAMPADD | head 1 - | """.stripMargin)) - } - - test("test TIMESTAMPDIFF is not supported") { - intercept[Exception](sql(s""" - | source = $testTable - | | eval `TIMESTAMPDIFF_1` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') - | | fields TIMESTAMPDIFF_1 | head 1 - | """.stripMargin)) - } - test("test TO_DAYS is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" | source = $testTable @@ -497,15 +574,6 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assert(ex.getMessage.contains("UTC_TIME is not a builtin function of PPL")) } - test("test UTC_TIMESTAMP is not supported") { - val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable - | | eval `UTC_TIMESTAMP` = UTC_TIMESTAMP() - | | fields UTC_TIMESTAMP | head 1 - | """.stripMargin)) - assert(ex.getMessage.contains("UTC_TIMESTAMP is not a builtin function of PPL")) - } - test("test YEARWEEK is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" | source = $testTable diff --git a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 index 82fdeb42f..b48cc524b 100644 --- a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -295,6 +295,7 @@ CURDATE: 'CURDATE'; CURRENT_DATE: 'CURRENT_DATE'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; +CURRENT_TIMEZONE: 'CURRENT_TIMEZONE'; CURTIME: 'CURTIME'; DATE: 'DATE'; DATEDIFF: 'DATEDIFF'; diff --git a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.tokens b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.tokens new file mode 100644 index 000000000..60029f9ca --- /dev/null +++ b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.tokens @@ -0,0 +1,764 @@ +SEARCH=1 +DESCRIBE=2 +SHOW=3 +FROM=4 +WHERE=5 +FIELDS=6 +RENAME=7 +STATS=8 +EVENTSTATS=9 +DEDUP=10 +SORT=11 +EVAL=12 +HEAD=13 +TOP=14 +RARE=15 +PARSE=16 +METHOD=17 +REGEX=18 +PUNCT=19 +GROK=20 +PATTERN=21 +PATTERNS=22 +NEW_FIELD=23 +KMEANS=24 +AD=25 +ML=26 +FILLNULL=27 +JOIN=28 +ON=29 +INNER=30 +OUTER=31 +FULL=32 +SEMI=33 +ANTI=34 +CROSS=35 +LEFT_HINT=36 +RIGHT_HINT=37 +CORRELATE=38 +SELF=39 +EXACT=40 +APPROXIMATE=41 +SCOPE=42 +MAPPING=43 +EXPLAIN=44 +FORMATTED=45 +COST=46 +CODEGEN=47 +EXTENDED=48 +SIMPLE=49 +AS=50 +BY=51 +SOURCE=52 +INDEX=53 +D=54 +DESC=55 +DATASOURCES=56 +USING=57 +WITH=58 +AUTO=59 +STR=60 +IP=61 +NUM=62 +FIELDSUMMARY=63 +INCLUDEFIELDS=64 +NULLS=65 +KEEPEMPTY=66 +CONSECUTIVE=67 +DEDUP_SPLITVALUES=68 +PARTITIONS=69 +ALLNUM=70 +DELIM=71 +CENTROIDS=72 +ITERATIONS=73 +DISTANCE_TYPE=74 +NUMBER_OF_TREES=75 +SHINGLE_SIZE=76 +SAMPLE_SIZE=77 +OUTPUT_AFTER=78 +TIME_DECAY=79 +ANOMALY_RATE=80 +CATEGORY_FIELD=81 +TIME_FIELD=82 +TIME_ZONE=83 +TRAINING_DATA_SIZE=84 +ANOMALY_SCORE_THRESHOLD=85 +APPEND=86 +CASE=87 +ELSE=88 +IN=89 +EXISTS=90 +NOT=91 +OR=92 +AND=93 +XOR=94 +TRUE=95 +FALSE=96 +REGEXP=97 +CONVERT_TZ=98 +DATETIME=99 +DAY=100 +DAY_HOUR=101 +DAY_MICROSECOND=102 +DAY_MINUTE=103 +DAY_OF_YEAR=104 +DAY_SECOND=105 +HOUR=106 +HOUR_MICROSECOND=107 +HOUR_MINUTE=108 +HOUR_OF_DAY=109 +HOUR_SECOND=110 +INTERVAL=111 +MICROSECOND=112 +MILLISECOND=113 +MINUTE=114 +MINUTE_MICROSECOND=115 +MINUTE_OF_DAY=116 +MINUTE_OF_HOUR=117 +MINUTE_SECOND=118 +MONTH=119 +MONTH_OF_YEAR=120 +QUARTER=121 +SECOND=122 +SECOND_MICROSECOND=123 +SECOND_OF_MINUTE=124 +WEEK=125 +WEEK_OF_YEAR=126 +YEAR=127 +YEAR_MONTH=128 +DATAMODEL=129 +LOOKUP=130 +SAVEDSEARCH=131 +INT=132 +INTEGER=133 +DOUBLE=134 +LONG=135 +FLOAT=136 +STRING=137 +BOOLEAN=138 +PIPE=139 +COMMA=140 +DOT=141 +EQUAL=142 +GREATER=143 +LESS=144 +NOT_GREATER=145 +NOT_LESS=146 +NOT_EQUAL=147 +PLUS=148 +MINUS=149 +STAR=150 +DIVIDE=151 +MODULE=152 +EXCLAMATION_SYMBOL=153 +COLON=154 +LT_PRTHS=155 +RT_PRTHS=156 +LT_SQR_PRTHS=157 +RT_SQR_PRTHS=158 +SINGLE_QUOTE=159 +DOUBLE_QUOTE=160 +BACKTICK=161 +BIT_NOT_OP=162 +BIT_AND_OP=163 +BIT_XOR_OP=164 +AVG=165 +COUNT=166 +DISTINCT_COUNT=167 +ESTDC=168 +ESTDC_ERROR=169 +MAX=170 +MEAN=171 +MEDIAN=172 +MIN=173 +MODE=174 +RANGE=175 +STDEV=176 +STDEVP=177 +SUM=178 +SUMSQ=179 +VAR_SAMP=180 +VAR_POP=181 +STDDEV_SAMP=182 +STDDEV_POP=183 +PERCENTILE=184 +PERCENTILE_APPROX=185 +TAKE=186 +FIRST=187 +LAST=188 +LIST=189 +VALUES=190 +EARLIEST=191 +EARLIEST_TIME=192 +LATEST=193 +LATEST_TIME=194 +PER_DAY=195 +PER_HOUR=196 +PER_MINUTE=197 +PER_SECOND=198 +RATE=199 +SPARKLINE=200 +C=201 +DC=202 +ABS=203 +CBRT=204 +CEIL=205 +CEILING=206 +CONV=207 +CRC32=208 +E=209 +EXP=210 +FLOOR=211 +LN=212 +LOG=213 +LOG10=214 +LOG2=215 +MOD=216 +PI=217 +POSITION=218 +POW=219 +POWER=220 +RAND=221 +ROUND=222 +SIGN=223 +SIGNUM=224 +SQRT=225 +TRUNCATE=226 +ACOS=227 +ASIN=228 +ATAN=229 +ATAN2=230 +COS=231 +COT=232 +DEGREES=233 +RADIANS=234 +SIN=235 +TAN=236 +MD5=237 +SHA1=238 +SHA2=239 +ADDDATE=240 +ADDTIME=241 +CURDATE=242 +CURRENT_DATE=243 +CURRENT_TIME=244 +CURRENT_TIMESTAMP=245 +CURRENT_TIMEZONE=246 +CURTIME=247 +DATE=248 +DATEDIFF=249 +DATE_ADD=250 +DATE_FORMAT=251 +DATE_SUB=252 +DAYNAME=253 +DAYOFMONTH=254 +DAYOFWEEK=255 +DAYOFYEAR=256 +DAY_OF_MONTH=257 +DAY_OF_WEEK=258 +DURATION=259 +EXTRACT=260 +FROM_DAYS=261 +FROM_UNIXTIME=262 +GET_FORMAT=263 +LAST_DAY=264 +LOCALTIME=265 +LOCALTIMESTAMP=266 +MAKEDATE=267 +MAKE_DATE=268 +MAKETIME=269 +MONTHNAME=270 +NOW=271 +PERIOD_ADD=272 +PERIOD_DIFF=273 +SEC_TO_TIME=274 +STR_TO_DATE=275 +SUBDATE=276 +SUBTIME=277 +SYSDATE=278 +TIME=279 +TIMEDIFF=280 +TIMESTAMP=281 +TIMESTAMPADD=282 +TIMESTAMPDIFF=283 +TIME_FORMAT=284 +TIME_TO_SEC=285 +TO_DAYS=286 +TO_SECONDS=287 +UNIX_TIMESTAMP=288 +UTC_DATE=289 +UTC_TIME=290 +UTC_TIMESTAMP=291 +WEEKDAY=292 +YEARWEEK=293 +SUBSTR=294 +SUBSTRING=295 +LTRIM=296 +RTRIM=297 +TRIM=298 +TO=299 +LOWER=300 +UPPER=301 +CONCAT=302 +CONCAT_WS=303 +LENGTH=304 +STRCMP=305 +RIGHT=306 +LEFT=307 +ASCII=308 +LOCATE=309 +REPLACE=310 +REVERSE=311 +CAST=312 +ISEMPTY=313 +ISBLANK=314 +JSON=315 +JSON_OBJECT=316 +JSON_ARRAY=317 +JSON_ARRAY_LENGTH=318 +JSON_EXTRACT=319 +JSON_KEYS=320 +JSON_VALID=321 +ARRAY=322 +LIKE=323 +ISNULL=324 +ISNOTNULL=325 +ISPRESENT=326 +BETWEEN=327 +IFNULL=328 +NULLIF=329 +IF=330 +TYPEOF=331 +COALESCE=332 +MATCH=333 +MATCH_PHRASE=334 +MATCH_PHRASE_PREFIX=335 +MATCH_BOOL_PREFIX=336 +SIMPLE_QUERY_STRING=337 +MULTI_MATCH=338 +QUERY_STRING=339 +ALLOW_LEADING_WILDCARD=340 +ANALYZE_WILDCARD=341 +ANALYZER=342 +AUTO_GENERATE_SYNONYMS_PHRASE_QUERY=343 +BOOST=344 +CUTOFF_FREQUENCY=345 +DEFAULT_FIELD=346 +DEFAULT_OPERATOR=347 +ENABLE_POSITION_INCREMENTS=348 +ESCAPE=349 +FLAGS=350 +FUZZY_MAX_EXPANSIONS=351 +FUZZY_PREFIX_LENGTH=352 +FUZZY_TRANSPOSITIONS=353 +FUZZY_REWRITE=354 +FUZZINESS=355 +LENIENT=356 +LOW_FREQ_OPERATOR=357 +MAX_DETERMINIZED_STATES=358 +MAX_EXPANSIONS=359 +MINIMUM_SHOULD_MATCH=360 +OPERATOR=361 +PHRASE_SLOP=362 +PREFIX_LENGTH=363 +QUOTE_ANALYZER=364 +QUOTE_FIELD_SUFFIX=365 +REWRITE=366 +SLOP=367 +TIE_BREAKER=368 +TYPE=369 +ZERO_TERMS_QUERY=370 +SPAN=371 +MS=372 +S=373 +M=374 +H=375 +W=376 +Q=377 +Y=378 +ID=379 +CLUSTER=380 +INTEGER_LITERAL=381 +DECIMAL_LITERAL=382 +ID_DATE_SUFFIX=383 +DQUOTA_STRING=384 +SQUOTA_STRING=385 +BQUOTA_STRING=386 +LINE_COMMENT=387 +BLOCK_COMMENT=388 +ERROR_RECOGNITION=389 +'SEARCH'=1 +'DESCRIBE'=2 +'SHOW'=3 +'FROM'=4 +'WHERE'=5 +'FIELDS'=6 +'RENAME'=7 +'STATS'=8 +'EVENTSTATS'=9 +'DEDUP'=10 +'SORT'=11 +'EVAL'=12 +'HEAD'=13 +'TOP'=14 +'RARE'=15 +'PARSE'=16 +'METHOD'=17 +'REGEX'=18 +'PUNCT'=19 +'GROK'=20 +'PATTERN'=21 +'PATTERNS'=22 +'NEW_FIELD'=23 +'KMEANS'=24 +'AD'=25 +'ML'=26 +'FILLNULL'=27 +'JOIN'=28 +'ON'=29 +'INNER'=30 +'OUTER'=31 +'FULL'=32 +'SEMI'=33 +'ANTI'=34 +'CROSS'=35 +'HINT.LEFT'=36 +'HINT.RIGHT'=37 +'CORRELATE'=38 +'SELF'=39 +'EXACT'=40 +'APPROXIMATE'=41 +'SCOPE'=42 +'MAPPING'=43 +'EXPLAIN'=44 +'FORMATTED'=45 +'COST'=46 +'CODEGEN'=47 +'EXTENDED'=48 +'SIMPLE'=49 +'AS'=50 +'BY'=51 +'SOURCE'=52 +'INDEX'=53 +'D'=54 +'DESC'=55 +'DATASOURCES'=56 +'USING'=57 +'WITH'=58 +'AUTO'=59 +'STR'=60 +'IP'=61 +'NUM'=62 +'FIELDSUMMARY'=63 +'INCLUDEFIELDS'=64 +'NULLS'=65 +'KEEPEMPTY'=66 +'CONSECUTIVE'=67 +'DEDUP_SPLITVALUES'=68 +'PARTITIONS'=69 +'ALLNUM'=70 +'DELIM'=71 +'CENTROIDS'=72 +'ITERATIONS'=73 +'DISTANCE_TYPE'=74 +'NUMBER_OF_TREES'=75 +'SHINGLE_SIZE'=76 +'SAMPLE_SIZE'=77 +'OUTPUT_AFTER'=78 +'TIME_DECAY'=79 +'ANOMALY_RATE'=80 +'CATEGORY_FIELD'=81 +'TIME_FIELD'=82 +'TIME_ZONE'=83 +'TRAINING_DATA_SIZE'=84 +'ANOMALY_SCORE_THRESHOLD'=85 +'APPEND'=86 +'CASE'=87 +'ELSE'=88 +'IN'=89 +'EXISTS'=90 +'NOT'=91 +'OR'=92 +'AND'=93 +'XOR'=94 +'TRUE'=95 +'FALSE'=96 +'REGEXP'=97 +'CONVERT_TZ'=98 +'DATETIME'=99 +'DAY'=100 +'DAY_HOUR'=101 +'DAY_MICROSECOND'=102 +'DAY_MINUTE'=103 +'DAY_OF_YEAR'=104 +'DAY_SECOND'=105 +'HOUR'=106 +'HOUR_MICROSECOND'=107 +'HOUR_MINUTE'=108 +'HOUR_OF_DAY'=109 +'HOUR_SECOND'=110 +'INTERVAL'=111 +'MICROSECOND'=112 +'MILLISECOND'=113 +'MINUTE'=114 +'MINUTE_MICROSECOND'=115 +'MINUTE_OF_DAY'=116 +'MINUTE_OF_HOUR'=117 +'MINUTE_SECOND'=118 +'MONTH'=119 +'MONTH_OF_YEAR'=120 +'QUARTER'=121 +'SECOND'=122 +'SECOND_MICROSECOND'=123 +'SECOND_OF_MINUTE'=124 +'WEEK'=125 +'WEEK_OF_YEAR'=126 +'YEAR'=127 +'YEAR_MONTH'=128 +'DATAMODEL'=129 +'LOOKUP'=130 +'SAVEDSEARCH'=131 +'INT'=132 +'INTEGER'=133 +'DOUBLE'=134 +'LONG'=135 +'FLOAT'=136 +'STRING'=137 +'BOOLEAN'=138 +'|'=139 +','=140 +'.'=141 +'='=142 +'>'=143 +'<'=144 +'+'=148 +'-'=149 +'*'=150 +'/'=151 +'%'=152 +'!'=153 +':'=154 +'('=155 +')'=156 +'['=157 +']'=158 +'\''=159 +'"'=160 +'`'=161 +'~'=162 +'&'=163 +'^'=164 +'AVG'=165 +'COUNT'=166 +'DISTINCT_COUNT'=167 +'ESTDC'=168 +'ESTDC_ERROR'=169 +'MAX'=170 +'MEAN'=171 +'MEDIAN'=172 +'MIN'=173 +'MODE'=174 +'RANGE'=175 +'STDEV'=176 +'STDEVP'=177 +'SUM'=178 +'SUMSQ'=179 +'VAR_SAMP'=180 +'VAR_POP'=181 +'STDDEV_SAMP'=182 +'STDDEV_POP'=183 +'PERCENTILE'=184 +'PERCENTILE_APPROX'=185 +'TAKE'=186 +'FIRST'=187 +'LAST'=188 +'LIST'=189 +'VALUES'=190 +'EARLIEST'=191 +'EARLIEST_TIME'=192 +'LATEST'=193 +'LATEST_TIME'=194 +'PER_DAY'=195 +'PER_HOUR'=196 +'PER_MINUTE'=197 +'PER_SECOND'=198 +'RATE'=199 +'SPARKLINE'=200 +'C'=201 +'DC'=202 +'ABS'=203 +'CBRT'=204 +'CEIL'=205 +'CEILING'=206 +'CONV'=207 +'CRC32'=208 +'E'=209 +'EXP'=210 +'FLOOR'=211 +'LN'=212 +'LOG'=213 +'LOG10'=214 +'LOG2'=215 +'MOD'=216 +'PI'=217 +'POSITION'=218 +'POW'=219 +'POWER'=220 +'RAND'=221 +'ROUND'=222 +'SIGN'=223 +'SIGNUM'=224 +'SQRT'=225 +'TRUNCATE'=226 +'ACOS'=227 +'ASIN'=228 +'ATAN'=229 +'ATAN2'=230 +'COS'=231 +'COT'=232 +'DEGREES'=233 +'RADIANS'=234 +'SIN'=235 +'TAN'=236 +'MD5'=237 +'SHA1'=238 +'SHA2'=239 +'ADDDATE'=240 +'ADDTIME'=241 +'CURDATE'=242 +'CURRENT_DATE'=243 +'CURRENT_TIME'=244 +'CURRENT_TIMESTAMP'=245 +'CURRENT_TIMEZONE'=246 +'CURTIME'=247 +'DATE'=248 +'DATEDIFF'=249 +'DATE_ADD'=250 +'DATE_FORMAT'=251 +'DATE_SUB'=252 +'DAYNAME'=253 +'DAYOFMONTH'=254 +'DAYOFWEEK'=255 +'DAYOFYEAR'=256 +'DAY_OF_MONTH'=257 +'DAY_OF_WEEK'=258 +'DURATION'=259 +'EXTRACT'=260 +'FROM_DAYS'=261 +'FROM_UNIXTIME'=262 +'GET_FORMAT'=263 +'LAST_DAY'=264 +'LOCALTIME'=265 +'LOCALTIMESTAMP'=266 +'MAKEDATE'=267 +'MAKE_DATE'=268 +'MAKETIME'=269 +'MONTHNAME'=270 +'NOW'=271 +'PERIOD_ADD'=272 +'PERIOD_DIFF'=273 +'SEC_TO_TIME'=274 +'STR_TO_DATE'=275 +'SUBDATE'=276 +'SUBTIME'=277 +'SYSDATE'=278 +'TIME'=279 +'TIMEDIFF'=280 +'TIMESTAMP'=281 +'TIMESTAMPADD'=282 +'TIMESTAMPDIFF'=283 +'TIME_FORMAT'=284 +'TIME_TO_SEC'=285 +'TO_DAYS'=286 +'TO_SECONDS'=287 +'UNIX_TIMESTAMP'=288 +'UTC_DATE'=289 +'UTC_TIME'=290 +'UTC_TIMESTAMP'=291 +'WEEKDAY'=292 +'YEARWEEK'=293 +'SUBSTR'=294 +'SUBSTRING'=295 +'LTRIM'=296 +'RTRIM'=297 +'TRIM'=298 +'TO'=299 +'LOWER'=300 +'UPPER'=301 +'CONCAT'=302 +'CONCAT_WS'=303 +'LENGTH'=304 +'STRCMP'=305 +'RIGHT'=306 +'LEFT'=307 +'ASCII'=308 +'LOCATE'=309 +'REPLACE'=310 +'REVERSE'=311 +'CAST'=312 +'ISEMPTY'=313 +'ISBLANK'=314 +'JSON'=315 +'JSON_OBJECT'=316 +'JSON_ARRAY'=317 +'JSON_ARRAY_LENGTH'=318 +'JSON_EXTRACT'=319 +'JSON_KEYS'=320 +'JSON_VALID'=321 +'ARRAY'=322 +'LIKE'=323 +'ISNULL'=324 +'ISNOTNULL'=325 +'ISPRESENT'=326 +'BETWEEN'=327 +'IFNULL'=328 +'NULLIF'=329 +'IF'=330 +'TYPEOF'=331 +'COALESCE'=332 +'MATCH'=333 +'MATCH_PHRASE'=334 +'MATCH_PHRASE_PREFIX'=335 +'MATCH_BOOL_PREFIX'=336 +'SIMPLE_QUERY_STRING'=337 +'MULTI_MATCH'=338 +'QUERY_STRING'=339 +'ALLOW_LEADING_WILDCARD'=340 +'ANALYZE_WILDCARD'=341 +'ANALYZER'=342 +'AUTO_GENERATE_SYNONYMS_PHRASE_QUERY'=343 +'BOOST'=344 +'CUTOFF_FREQUENCY'=345 +'DEFAULT_FIELD'=346 +'DEFAULT_OPERATOR'=347 +'ENABLE_POSITION_INCREMENTS'=348 +'ESCAPE'=349 +'FLAGS'=350 +'FUZZY_MAX_EXPANSIONS'=351 +'FUZZY_PREFIX_LENGTH'=352 +'FUZZY_TRANSPOSITIONS'=353 +'FUZZY_REWRITE'=354 +'FUZZINESS'=355 +'LENIENT'=356 +'LOW_FREQ_OPERATOR'=357 +'MAX_DETERMINIZED_STATES'=358 +'MAX_EXPANSIONS'=359 +'MINIMUM_SHOULD_MATCH'=360 +'OPERATOR'=361 +'PHRASE_SLOP'=362 +'PREFIX_LENGTH'=363 +'QUOTE_ANALYZER'=364 +'QUOTE_FIELD_SUFFIX'=365 +'REWRITE'=366 +'SLOP'=367 +'TIE_BREAKER'=368 +'TYPE'=369 +'ZERO_TERMS_QUERY'=370 +'SPAN'=371 +'MS'=372 +'S'=373 +'M'=374 +'H'=375 +'W'=376 +'Q'=377 +'Y'=378 diff --git a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 index 48984b3a5..2b1632e50 100644 --- a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 @@ -421,6 +421,7 @@ valueExpression | primaryExpression # valueExpressionDefault | positionFunction # positionFunctionCall | caseFunction # caseExpr + | timestampFunction # timestampFunctionCall | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr ; @@ -672,6 +673,7 @@ dateTimeFunctionName | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP + | CURRENT_TIMEZONE | CURTIME | DATE | DATEDIFF @@ -888,6 +890,7 @@ literalValue | decimalLiteral | booleanLiteral | datetimeLiteral //#datetime + | intervalLiteral ; intervalLiteral diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Interval.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Interval.java index fc09ec2f5..bf00b2106 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Interval.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Interval.java @@ -23,11 +23,6 @@ public class Interval extends UnresolvedExpression { private final UnresolvedExpression value; private final IntervalUnit unit; - public Interval(UnresolvedExpression value, String unit) { - this.value = value; - this.unit = IntervalUnit.of(unit); - } - @Override public List getChild() { return Collections.singletonList(value); diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java index a7e983473..6e1e0712c 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java @@ -17,6 +17,7 @@ public enum IntervalUnit { UNKNOWN, MICROSECOND, + MILLISECOND, SECOND, MINUTE, HOUR, diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 9e1a9a743..d81dc7ce4 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -64,9 +64,9 @@ public enum BuiltinFunctionName { DATE(FunctionName.of("date")), DATEDIFF(FunctionName.of("datediff")), // DATETIME(FunctionName.of("datetime")), -// DATE_ADD(FunctionName.of("date_add")), + DATE_ADD(FunctionName.of("date_add")), DATE_FORMAT(FunctionName.of("date_format")), -// DATE_SUB(FunctionName.of("date_sub")), + DATE_SUB(FunctionName.of("date_sub")), DAY(FunctionName.of("day")), // DAYNAME(FunctionName.of("dayname")), DAYOFMONTH(FunctionName.of("dayofmonth")), @@ -105,14 +105,15 @@ public enum BuiltinFunctionName { // TIMEDIFF(FunctionName.of("timediff")), // TIME_TO_SEC(FunctionName.of("time_to_sec")), TIMESTAMP(FunctionName.of("timestamp")), -// TIMESTAMPADD(FunctionName.of("timestampadd")), -// TIMESTAMPDIFF(FunctionName.of("timestampdiff")), + TIMESTAMPADD(FunctionName.of("timestampadd")), + TIMESTAMPDIFF(FunctionName.of("timestampdiff")), // TIME_FORMAT(FunctionName.of("time_format")), // TO_DAYS(FunctionName.of("to_days")), // TO_SECONDS(FunctionName.of("to_seconds")), // UTC_DATE(FunctionName.of("utc_date")), // UTC_TIME(FunctionName.of("utc_time")), -// UTC_TIMESTAMP(FunctionName.of("utc_timestamp")), + UTC_TIMESTAMP(FunctionName.of("utc_timestamp")), + CURRENT_TIMEZONE(FunctionName.of("current_timezone")), UNIX_TIMESTAMP(FunctionName.of("unix_timestamp")), WEEK(FunctionName.of("week")), WEEKDAY(FunctionName.of("weekday")), diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/CatalystQueryPlanVisitor.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/CatalystQueryPlanVisitor.java index 441287ddb..53f1b2aea 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/CatalystQueryPlanVisitor.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/CatalystQueryPlanVisitor.java @@ -20,12 +20,15 @@ import org.apache.spark.sql.catalyst.expressions.InSubquery$; import org.apache.spark.sql.catalyst.expressions.LessThanOrEqual; import org.apache.spark.sql.catalyst.expressions.ListQuery$; +import org.apache.spark.sql.catalyst.expressions.Literal$; +import org.apache.spark.sql.catalyst.expressions.MakeInterval$; import org.apache.spark.sql.catalyst.expressions.NamedExpression; import org.apache.spark.sql.catalyst.expressions.Predicate; import org.apache.spark.sql.catalyst.expressions.ScalarSubquery$; import org.apache.spark.sql.catalyst.expressions.SortDirection; import org.apache.spark.sql.catalyst.expressions.SortOrder; import org.apache.spark.sql.catalyst.plans.logical.*; +import org.apache.spark.sql.catalyst.util.DataTypeJsonUtils$; import org.apache.spark.sql.execution.ExplainMode; import org.apache.spark.sql.execution.command.DescribeTableCommand; import org.apache.spark.sql.execution.command.ExplainCommand; @@ -759,7 +762,91 @@ public Expression visitFillNull(FillNull fillNull, CatalystPlanContext context) @Override public Expression visitInterval(Interval node, CatalystPlanContext context) { - throw new IllegalStateException("Not Supported operation : Interval"); + node.getValue().accept(this, context); + Expression value = context.getNamedParseExpressions().pop(); + Expression interval; + switch (node.getUnit()) { + case YEAR: + interval = MakeInterval$.MODULE$.apply( + value, + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + true); + break; + case MONTH: + interval = MakeInterval$.MODULE$.apply( + Literal$.MODULE$.apply(0), + value, + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + true); + break; + case WEEK: + interval = MakeInterval$.MODULE$.apply( + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + value, + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + true); + break; + case DAY: + interval = MakeInterval$.MODULE$.apply( + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + value, + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + true); + break; + case HOUR: + interval = MakeInterval$.MODULE$.apply( + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + value, + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + true); + break; + case MINUTE: + interval = MakeInterval$.MODULE$.apply( + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + value, + Literal$.MODULE$.apply(0), + true); + break; + case SECOND: + interval = MakeInterval$.MODULE$.apply( + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + Literal$.MODULE$.apply(0), + value, + true); + break; + default: + throw new IllegalArgumentException("Unsupported Interval unit: " + node.getUnit()); + } + return context.getNamedParseExpressions().push(interval); } @Override diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 6a0c80c16..611dd3914 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -421,6 +421,23 @@ public UnresolvedExpression visitInExpr(OpenSearchPPLParser.InExprContext ctx) { return ctx.NOT() != null ? new Not(expr) : expr; } + @Override + public UnresolvedExpression visitTimestampFunctionCall( + OpenSearchPPLParser.TimestampFunctionCallContext ctx) { + return new Function( + ctx.timestampFunction().timestampFunctionName().getText(), timestampFunctionArguments(ctx)); + } + + private List timestampFunctionArguments( + OpenSearchPPLParser.TimestampFunctionCallContext ctx) { + List args = + Arrays.asList( + new Literal(ctx.timestampFunction().simpleDateTimePart().getText(), DataType.STRING), + visitFunctionArg(ctx.timestampFunction().firstArg), + visitFunctionArg(ctx.timestampFunction().secondArg)); + return args; + } + private QualifiedName visitIdentifiers(List ctx) { return new QualifiedName( ctx.stream() diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java index 8982fe859..2f44bb287 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTranslator.java @@ -8,9 +8,17 @@ import com.google.common.collect.ImmutableMap; import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction; import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction$; +import org.apache.spark.sql.catalyst.expressions.CurrentTimeZone$; +import org.apache.spark.sql.catalyst.expressions.CurrentTimestamp$; +import org.apache.spark.sql.catalyst.expressions.DateAddInterval$; import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.catalyst.expressions.Literal$; +import org.apache.spark.sql.catalyst.expressions.TimestampAdd$; +import org.apache.spark.sql.catalyst.expressions.TimestampDiff$; +import org.apache.spark.sql.catalyst.expressions.ToUTCTimestamp$; +import org.apache.spark.sql.catalyst.expressions.UnaryMinus$; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import scala.Option; import java.util.List; import java.util.Map; @@ -19,6 +27,8 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADD; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDDATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATEDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_SUB; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_MONTH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.COALESCE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON; @@ -44,7 +54,10 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND_OF_MINUTE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBDATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.SYSDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIMESTAMP; import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK; import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK_OF_YEAR; import static org.opensearch.sql.ppl.utils.DataTypeTransformer.seq; @@ -95,8 +108,8 @@ public interface BuiltinFunctionTranslator { /** * The name mapping between PPL builtin functions to Spark builtin functions. */ - static final Map, UnresolvedFunction>> PPL_TO_SPARK_FUNC_MAPPING - = ImmutableMap., UnresolvedFunction>>builder() + static final Map, Expression>> PPL_TO_SPARK_FUNC_MAPPING + = ImmutableMap., Expression>>builder() // json functions .put( JSON_ARRAY, @@ -139,6 +152,31 @@ public interface BuiltinFunctionTranslator { seq(UnresolvedFunction$.MODULE$.apply("get_json_object", seq(args.get(0), Literal$.MODULE$.apply("$")), false)), false); }) + .put( + DATE_ADD, + args -> { + return DateAddInterval$.MODULE$.apply(args.get(0), args.get(1), Option.empty(), false); + }) + .put( + DATE_SUB, + args -> { + return DateAddInterval$.MODULE$.apply(args.get(0), UnaryMinus$.MODULE$.apply(args.get(1), true), Option.empty(), true); + }) + .put( + TIMESTAMPADD, + args -> { + return TimestampAdd$.MODULE$.apply(args.get(0).toString(), args.get(1), args.get(2), Option.empty()); + }) + .put( + TIMESTAMPDIFF, + args -> { + return TimestampDiff$.MODULE$.apply(args.get(0).toString(), args.get(1), args.get(2), Option.empty()); + }) + .put( + UTC_TIMESTAMP, + args -> { + return ToUTCTimestamp$.MODULE$.apply(CurrentTimestamp$.MODULE$.apply(), CurrentTimeZone$.MODULE$.apply()); + }) .build(); static Expression builtinFunction(org.opensearch.sql.ast.expression.Function function, List args) { @@ -153,7 +191,7 @@ static Expression builtinFunction(org.opensearch.sql.ast.expression.Function fun // there is a Spark builtin function mapping with the PPL builtin function return new UnresolvedFunction(seq(name), seq(args), false, empty(),false); } - Function, UnresolvedFunction> alternative = PPL_TO_SPARK_FUNC_MAPPING.get(builtin); + Function, Expression> alternative = PPL_TO_SPARK_FUNC_MAPPING.get(builtin); if (alternative != null) { return alternative.apply(args); } diff --git a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanDateTimeFunctionsTranslatorTestSuite.scala b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanDateTimeFunctionsTranslatorTestSuite.scala new file mode 100644 index 000000000..308b038bb --- /dev/null +++ b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanDateTimeFunctionsTranslatorTestSuite.scala @@ -0,0 +1,231 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.flint.spark.ppl + +import org.opensearch.flint.spark.ppl.PlaneUtils.plan +import org.opensearch.sql.ppl.{CatalystPlanContext, CatalystQueryPlanVisitor} +import org.scalatest.matchers.should.Matchers + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.analysis.{UnresolvedFunction, UnresolvedRelation, UnresolvedStar} +import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentTimestamp, CurrentTimeZone, DateAddInterval, Literal, MakeInterval, NamedExpression, TimestampAdd, TimestampDiff, ToUTCTimestamp, UnaryMinus} +import org.apache.spark.sql.catalyst.plans.PlanTest +import org.apache.spark.sql.catalyst.plans.logical.Project + +class PPLLogicalPlanDateTimeFunctionsTranslatorTestSuite + extends SparkFunSuite + with PlanTest + with LogicalPlanTestUtils + with Matchers { + + private val planTransformer = new CatalystQueryPlanVisitor() + private val pplParser = new PPLSyntaxParser() + + test("test DATE_ADD") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan(pplParser, "source=t | eval a = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 DAY)"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + DateAddInterval( + UnresolvedFunction("date", Seq(Literal("2020-08-26")), isDistinct = false), + MakeInterval( + Literal(0), + Literal(0), + Literal(0), + Literal(2), + Literal(0), + Literal(0), + Literal(0), + failOnError = true)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test DATE_ADD for year") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan(pplParser, "source=t | eval a = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 YEAR)"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + DateAddInterval( + UnresolvedFunction("date", Seq(Literal("2020-08-26")), isDistinct = false), + MakeInterval( + Literal(2), + Literal(0), + Literal(0), + Literal(0), + Literal(0), + Literal(0), + Literal(0), + failOnError = true)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test DATE_SUB") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan(pplParser, "source=t | eval a = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 DAY)"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + DateAddInterval( + UnresolvedFunction("date", Seq(Literal("2020-08-26")), isDistinct = false), + UnaryMinus( + MakeInterval( + Literal(0), + Literal(0), + Literal(0), + Literal(2), + Literal(0), + Literal(0), + Literal(0), + failOnError = true), + failOnError = true)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + assert(compareByString(expectedPlan) === compareByString(logPlan)) + } + + test("test TIMESTAMPADD") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan(pplParser, "source=t | eval a = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias(TimestampAdd("DAY", Literal(17), Literal("2000-01-01 00:00:00")), "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test TIMESTAMPADD with timestamp") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan( + pplParser, + "source=t | eval a = TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00'))"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + TimestampAdd( + "DAY", + Literal(17), + UnresolvedFunction( + "timestamp", + Seq(Literal("2000-01-01 00:00:00")), + isDistinct = false)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test TIMESTAMPDIFF") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan( + pplParser, + "source=t | eval a = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + TimestampDiff("YEAR", Literal("1997-01-01 00:00:00"), Literal("2001-03-06 00:00:00")), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test TIMESTAMPDIFF with timestamp") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit( + plan( + pplParser, + "source=t | eval a = TIMESTAMPDIFF(YEAR, TIMESTAMP('1997-01-01 00:00:00'), TIMESTAMP('2001-03-06 00:00:00'))"), + context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias( + TimestampDiff( + "YEAR", + UnresolvedFunction( + "timestamp", + Seq(Literal("1997-01-01 00:00:00")), + isDistinct = false), + UnresolvedFunction( + "timestamp", + Seq(Literal("2001-03-06 00:00:00")), + isDistinct = false)), + "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test UTC_TIMESTAMP") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit(plan(pplParser, "source=t | eval a = UTC_TIMESTAMP()"), context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias(ToUTCTimestamp(CurrentTimestamp(), CurrentTimeZone()), "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } + + test("test CURRENT_TIMEZONE") { + val context = new CatalystPlanContext + val logPlan = + planTransformer.visit(plan(pplParser, "source=t | eval a = CURRENT_TIMEZONE()"), context) + + val table = UnresolvedRelation(Seq("t")) + val evalProjectList: Seq[NamedExpression] = Seq( + UnresolvedStar(None), + Alias(UnresolvedFunction("current_timezone", Seq.empty, isDistinct = false), "a")()) + val eval = Project(evalProjectList, table) + val expectedPlan = Project(Seq(UnresolvedStar(None)), eval) + comparePlans(expectedPlan, logPlan, checkAnalysis = false) + } +}