diff --git a/velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp b/velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp index 425cfd4879bb..1576be00c440 100644 --- a/velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp +++ b/velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp @@ -84,8 +84,8 @@ class DateTimeFunctionsTest : public functions::test::FunctionBaseTest { } public: - struct TimestampWithTimezone { - TimestampWithTimezone(int64_t milliSeconds, int16_t timezoneId) + struct TimestampWithTimezoneTest { + TimestampWithTimezoneTest(int64_t milliSeconds, int16_t timezoneId) : milliSeconds_(milliSeconds), timezoneId_(timezoneId) {} int64_t milliSeconds_{0}; @@ -94,83 +94,20 @@ class DateTimeFunctionsTest : public functions::test::FunctionBaseTest { // Provides a nicer printer for gtest. friend std::ostream& operator<<( std::ostream& os, - const TimestampWithTimezone& in) { + const TimestampWithTimezoneTest& in) { return os << "TimestampWithTimezone(milliSeconds: " << in.milliSeconds_ << ", timezoneId: " << in.timezoneId_ << ")"; } - }; - - std::optional parseDatetime( - const std::optional& input, - const std::optional& format) { - auto resultVector = evaluate( - "parse_datetime(c0, c1)", - makeRowVector( - {makeNullableFlatVector({input}), - makeNullableFlatVector({format})})); - EXPECT_EQ(1, resultVector->size()); - - if (resultVector->isNullAt(0)) { - return std::nullopt; - } - auto timestampWithTimezone = - resultVector->as>()->valueAt(0); - return TimestampWithTimezone{ - unpackMillisUtc(timestampWithTimezone), - unpackZoneKeyId(timestampWithTimezone)}; - } - - std::optional dateParse( - const std::optional& input, - const std::optional& format) { - auto resultVector = evaluate( - "date_parse(c0, c1)", - makeRowVector( - {makeNullableFlatVector({input}), - makeNullableFlatVector({format})})); - EXPECT_EQ(1, resultVector->size()); - - if (resultVector->isNullAt(0)) { - return std::nullopt; + static std::optional unpack( + const std::optional& timestampWithTimezone) { + return timestampWithTimezone + ? std::optional{TimestampWithTimezoneTest( + unpackMillisUtc(timestampWithTimezone.value()), + unpackZoneKeyId(timestampWithTimezone.value()))} + : std::nullopt; } - return resultVector->as>()->valueAt(0); - } - - std::optional dateFormat( - std::optional timestamp, - const std::string& format) { - auto resultVector = evaluate( - "date_format(c0, c1)", - makeRowVector( - {makeNullableFlatVector({timestamp}), - makeNullableFlatVector({format})})); - return resultVector->as>()->valueAt(0); - } - - std::optional formatDatetime( - std::optional timestamp, - const std::string& format) { - auto resultVector = evaluate( - "format_datetime(c0, c1)", - makeRowVector( - {makeNullableFlatVector({timestamp}), - makeNullableFlatVector({format})})); - return resultVector->as>()->valueAt(0); - } - - std::optional formatDatetimeWithTimezone( - std::optional timestamp, - std::optional timeZoneName, - const std::string& format) { - auto resultVector = evaluate( - "format_datetime(c0, c1)", - makeRowVector( - {makeTimestampWithTimeZoneVector( - timestamp.value().toMillis(), timeZoneName.value().c_str()), - makeNullableFlatVector({format})})); - return resultVector->as>()->valueAt(0); - } + }; template std::optional evaluateWithTimestampWithTimezone( @@ -242,8 +179,8 @@ class DateTimeFunctionsTest : public functions::test::FunctionBaseTest { }; bool operator==( - const DateTimeFunctionsTest::TimestampWithTimezone& a, - const DateTimeFunctionsTest::TimestampWithTimezone& b) { + const DateTimeFunctionsTest::TimestampWithTimezoneTest& a, + const DateTimeFunctionsTest::TimestampWithTimezoneTest& b) { return a.milliSeconds_ == b.milliSeconds_ && a.timezoneId_ == b.timezoneId_; } @@ -351,67 +288,31 @@ TEST_F(DateTimeFunctionsTest, fromUnixtimeRountTrip) { } TEST_F(DateTimeFunctionsTest, fromUnixtimeWithTimeZone) { - static const double kNan = std::numeric_limits::quiet_NaN(); - - vector_size_t size = 37; - - auto unixtimeAt = [](vector_size_t row) -> double { - return 1631800000.12345 + row * 11; + const auto fromUnixtime = [&](std::optional unixtime, + std::optional timezone) { + auto result = evaluateOnce( + "from_unixtime(c0, c1)", unixtime, timezone); + return TimestampWithTimezoneTest::unpack(result); }; - auto unixtimes = makeFlatVector(size, unixtimeAt); - - // Constant timezone parameter. - { - auto result = - evaluate("from_unixtime(c0, '+01:00')", makeRowVector({unixtimes})); - - auto expected = makeTimestampWithTimeZoneVector( - size, - [&](auto row) { return unixtimeAt(row) * 1'000; }, - [](auto /*row*/) { return 900; }); - assertEqualVectors(expected, result); - - // NaN timestamp. - result = evaluate( - "from_unixtime(c0, '+01:00')", - makeRowVector({makeFlatVector({kNan, kNan})})); - expected = makeTimestampWithTimeZoneVector( - 2, [](auto /*row*/) { return 0; }, [](auto /*row*/) { return 900; }); - assertEqualVectors(expected, result); - } - - // Variable timezone parameter. - { - std::vector timezoneIds = {900, 960, 1020, 1080, 1140}; - std::vector timezoneNames = { - "+01:00", "+02:00", "+03:00", "+04:00", "+05:00"}; + EXPECT_EQ( + TimestampWithTimezoneTest(0, util::getTimeZoneID("+01:00")), + fromUnixtime(0, "+01:00")); - auto timezones = makeFlatVector( - size, [&](auto row) { return StringView(timezoneNames[row % 5]); }); + EXPECT_EQ( + TimestampWithTimezoneTest( + 1000000, util::getTimeZoneID("America/Los_Angeles")), + fromUnixtime(1000, "America/Los_Angeles")); - auto result = evaluate( - "from_unixtime(c0, c1)", makeRowVector({unixtimes, timezones})); - auto expected = makeTimestampWithTimeZoneVector( - size, - [&](auto row) { return unixtimeAt(row) * 1'000; }, - [&](auto row) { return timezoneIds[row % 5]; }); - assertEqualVectors(expected, result); + EXPECT_EQ( + TimestampWithTimezoneTest(1631800000123, util::getTimeZoneID("-05:00")), + fromUnixtime(1631800000.12345, "-05:00")); - // NaN timestamp. - result = evaluate( - "from_unixtime(c0, c1)", - makeRowVector({ - makeFlatVector({kNan, kNan}), - makeNullableFlatVector({"+01:00", "+02:00"}), - })); - auto timezonesVector = std::vector{900, 960}; - expected = makeTimestampWithTimeZoneVector( - timezonesVector.size(), - [](auto /*row*/) { return 0; }, - [&](auto row) { return timezonesVector[row]; }); - assertEqualVectors(expected, result); - } + // NaN timestamp. + static const double kNan = std::numeric_limits::quiet_NaN(); + EXPECT_EQ( + TimestampWithTimezoneTest(0, util::getTimeZoneID("+01:00")), + fromUnixtime(kNan, "+01:00")); } TEST_F(DateTimeFunctionsTest, fromUnixtime) { @@ -2655,6 +2556,13 @@ TEST_F(DateTimeFunctionsTest, dateDiffTimestampWithTimezone) { } TEST_F(DateTimeFunctionsTest, parseDatetime) { + auto parseDatetime = [&](const std::optional& input, + const std::optional& format) { + auto timestampWithTimezone = evaluateOnce( + "parse_datetime(c0, c1)", input, format); + return TimestampWithTimezoneTest::unpack(timestampWithTimezone); + }; + // Check null behavior. EXPECT_EQ(std::nullopt, parseDatetime("1970-01-01", std::nullopt)); EXPECT_EQ(std::nullopt, parseDatetime(std::nullopt, "YYYY-MM-dd")); @@ -2669,42 +2577,47 @@ TEST_F(DateTimeFunctionsTest, parseDatetime) { // Simple tests. More exhaustive tests are provided as part of Joda's // implementation. EXPECT_EQ( - TimestampWithTimezone(0, 0), parseDatetime("1970-01-01", "YYYY-MM-dd")); + TimestampWithTimezoneTest(0, 0), + parseDatetime("1970-01-01", "YYYY-MM-dd")); EXPECT_EQ( - TimestampWithTimezone(86400000, 0), + TimestampWithTimezoneTest(86400000, 0), parseDatetime("1970-01-02", "YYYY-MM-dd")); EXPECT_EQ( - TimestampWithTimezone(86400000, 0), + TimestampWithTimezoneTest(86400000, 0), parseDatetime("19700102", "YYYYMMdd")); EXPECT_EQ( - TimestampWithTimezone(86400000, 0), parseDatetime("19700102", "YYYYMdd")); + TimestampWithTimezoneTest(86400000, 0), + parseDatetime("19700102", "YYYYMdd")); EXPECT_EQ( - TimestampWithTimezone(86400000, 0), parseDatetime("19700102", "YYYYMMd")); + TimestampWithTimezoneTest(86400000, 0), + parseDatetime("19700102", "YYYYMMd")); EXPECT_EQ( - TimestampWithTimezone(86400000, 0), parseDatetime("19700102", "YYYYMd")); + TimestampWithTimezoneTest(86400000, 0), + parseDatetime("19700102", "YYYYMd")); EXPECT_EQ( - TimestampWithTimezone(86400000, 0), parseDatetime("19700102", "YYYYMd")); + TimestampWithTimezoneTest(86400000, 0), + parseDatetime("19700102", "YYYYMd")); // 118860000 is the number of milliseconds since epoch at 1970-01-02 // 09:01:00.000 UTC. EXPECT_EQ( - TimestampWithTimezone(118860000, util::getTimeZoneID("+00:00")), + TimestampWithTimezoneTest(118860000, util::getTimeZoneID("+00:00")), parseDatetime("1970-01-02+09:01+00:00", "YYYY-MM-dd+HH:mmZZ")); EXPECT_EQ( - TimestampWithTimezone(118860000, util::getTimeZoneID("-09:00")), + TimestampWithTimezoneTest(118860000, util::getTimeZoneID("-09:00")), parseDatetime("1970-01-02+00:01-09:00", "YYYY-MM-dd+HH:mmZZ")); EXPECT_EQ( - TimestampWithTimezone(118860000, util::getTimeZoneID("-02:00")), + TimestampWithTimezoneTest(118860000, util::getTimeZoneID("-02:00")), parseDatetime("1970-01-02+07:01-02:00", "YYYY-MM-dd+HH:mmZZ")); EXPECT_EQ( - TimestampWithTimezone(118860000, util::getTimeZoneID("+14:00")), + TimestampWithTimezoneTest(118860000, util::getTimeZoneID("+14:00")), parseDatetime("1970-01-02+23:01+14:00", "YYYY-MM-dd+HH:mmZZ")); EXPECT_EQ( - TimestampWithTimezone( + TimestampWithTimezoneTest( 198060000, util::getTimeZoneID("America/Los_Angeles")), parseDatetime("1970-01-02+23:01 PST", "YYYY-MM-dd+HH:mm z")); EXPECT_EQ( - TimestampWithTimezone(169260000, util::getTimeZoneID("+00:00")), + TimestampWithTimezoneTest(169260000, util::getTimeZoneID("+00:00")), parseDatetime("1970-01-02+23:01 GMT", "YYYY-MM-dd+HH:mm z")); setQueryTimeZone("Asia/Kolkata"); @@ -2712,23 +2625,24 @@ TEST_F(DateTimeFunctionsTest, parseDatetime) { // 66600000 is the number of millisecond since epoch at 1970-01-01 // 18:30:00.000 UTC. EXPECT_EQ( - TimestampWithTimezone(66600000, util::getTimeZoneID("Asia/Kolkata")), + TimestampWithTimezoneTest(66600000, util::getTimeZoneID("Asia/Kolkata")), parseDatetime("1970-01-02+00:00", "YYYY-MM-dd+HH:mm")); EXPECT_EQ( - TimestampWithTimezone(66600000, util::getTimeZoneID("-03:00")), + TimestampWithTimezoneTest(66600000, util::getTimeZoneID("-03:00")), parseDatetime("1970-01-01+15:30-03:00", "YYYY-MM-dd+HH:mmZZ")); // -66600000 is the number of millisecond since epoch at 1969-12-31 // 05:30:00.000 UTC. EXPECT_EQ( - TimestampWithTimezone(-66600000, util::getTimeZoneID("Asia/Kolkata")), + TimestampWithTimezoneTest(-66600000, util::getTimeZoneID("Asia/Kolkata")), parseDatetime("1969-12-31+11:00", "YYYY-MM-dd+HH:mm")); EXPECT_EQ( - TimestampWithTimezone(-66600000, util::getTimeZoneID("+02:00")), + TimestampWithTimezoneTest(-66600000, util::getTimeZoneID("+02:00")), parseDatetime("1969-12-31+07:30+02:00", "YYYY-MM-dd+HH:mmZZ")); // Joda also lets 'Z' to be UTC|UCT|GMT|GMT0. - auto ts = TimestampWithTimezone(1708840800000, util::getTimeZoneID("GMT")); + auto ts = + TimestampWithTimezoneTest(1708840800000, util::getTimeZoneID("GMT")); EXPECT_EQ( ts, parseDatetime("2024-02-25+06:00:99 GMT", "yyyy-MM-dd+HH:mm:99 ZZZ")); EXPECT_EQ( @@ -2746,6 +2660,12 @@ TEST_F(DateTimeFunctionsTest, parseDatetime) { TEST_F(DateTimeFunctionsTest, formatDateTime) { using util::fromTimestampString; + auto formatDatetime = [&](const std::optional& timestamp, + const std::optional& format) { + return evaluateOnce( + "format_datetime(c0, c1)", timestamp, format); + }; + // era test cases - 'G' EXPECT_EQ("AD", formatDatetime(fromTimestampString("1970-01-01"), "G")); EXPECT_EQ("BC", formatDatetime(fromTimestampString("-100-01-01"), "G")); @@ -2994,12 +2914,16 @@ TEST_F(DateTimeFunctionsTest, formatDateTime) { formatDatetime(fromTimestampString("2022-01-01 03:30:30.001"), "SSSS")); // time zone test cases - 'z' - setQueryTimeZone("Asia/Kolkata"); + auto zeroTs = fromTimestampString("1970-01-01"); + + // No timezone set; default to GMT. EXPECT_EQ( - "Asia/Kolkata", - formatDatetime(fromTimestampString("1970-01-01"), "zzzz")); + "1970-01-01 00:00:00", formatDatetime(zeroTs, "YYYY-MM-dd HH:mm:ss")); - // literal test cases + setQueryTimeZone("Asia/Kolkata"); + EXPECT_EQ("Asia/Kolkata", formatDatetime(zeroTs, "zzzz")); + + // Literal test cases. EXPECT_EQ( "hello", formatDatetime(fromTimestampString("1970-01-01"), "'hello'")); EXPECT_EQ("'", formatDatetime(fromTimestampString("1970-01-01"), "''")); @@ -3054,9 +2978,14 @@ TEST_F(DateTimeFunctionsTest, formatDateTimeTimezone) { using util::fromTimestampString; auto zeroTs = fromTimestampString("1970-01-01"); - // No timezone set; default to GMT. - EXPECT_EQ( - "1970-01-01 00:00:00", formatDatetime(zeroTs, "YYYY-MM-dd HH:mm:ss")); + const auto formatDatetimeWithTimezone = [&](const Timestamp& timestamp, + const char* tz, + const std::string& format) { + auto input = makeTimestampWithTimeZoneVector(timestamp.toMillis(), tz); + return evaluateOnce( + "format_datetime(c0, c1)", + makeRowVector({input, makeNullableFlatVector({format})})); + }; // Check that string is adjusted to the timezone set. EXPECT_EQ( @@ -3071,15 +3000,12 @@ TEST_F(DateTimeFunctionsTest, formatDateTimeTimezone) { } TEST_F(DateTimeFunctionsTest, dateFormat) { - const auto dateFormatOnce = [&](std::optional timestamp, - const std::string& formatString) { - return evaluateOnce( - fmt::format("date_format(c0, '{}')", formatString), timestamp); - }; using util::fromTimestampString; - // Check null behaviors - EXPECT_EQ(std::nullopt, dateFormatOnce(std::nullopt, "%Y")); + auto dateFormat = [&](const std::optional& timestamp, + const std::optional& format) { + return evaluateOnce("date_format(c0, c1)", timestamp, format); + }; // Normal cases EXPECT_EQ( @@ -3369,6 +3295,17 @@ TEST_F(DateTimeFunctionsTest, dateFormat) { "Date format specifier is not supported: WEEK_YEAR"); } +TEST_F(DateTimeFunctionsTest, dateFormatWithConstant) { + const auto dateFormatOnce = [&](std::optional timestamp, + const std::string& formatString) { + return evaluateOnce( + fmt::format("date_format(c0, '{}')", formatString), timestamp); + }; + + // Check null behaviors + EXPECT_EQ(std::nullopt, dateFormatOnce(std::nullopt, "%Y")); +} + TEST_F(DateTimeFunctionsTest, dateFormatTimestampWithTimezone) { const auto testDateFormat = [&](const std::string& formatString, @@ -3428,6 +3365,11 @@ TEST_F(DateTimeFunctionsTest, fromIso8601Date) { } TEST_F(DateTimeFunctionsTest, dateParse) { + auto dateParse = [&](const std::optional& input, + const std::optional& format) { + return evaluateOnce("date_parse(c0, c1)", input, format); + }; + // Check null behavior. EXPECT_EQ(std::nullopt, dateParse("1970-01-01", std::nullopt)); EXPECT_EQ(std::nullopt, dateParse(std::nullopt, "YYYY-MM-dd")); diff --git a/velox/functions/prestosql/tests/utils/FunctionBaseTest.h b/velox/functions/prestosql/tests/utils/FunctionBaseTest.h index 8cedaec8e7d7..239c60459ad9 100644 --- a/velox/functions/prestosql/tests/utils/FunctionBaseTest.h +++ b/velox/functions/prestosql/tests/utils/FunctionBaseTest.h @@ -20,6 +20,7 @@ #include "velox/expression/Expr.h" #include "velox/parse/Expressions.h" #include "velox/parse/ExpressionsParser.h" +#include "velox/type/SimpleFunctionApi.h" #include "velox/type/Type.h" #include "velox/vector/tests/utils/VectorTestBase.h" @@ -159,6 +160,10 @@ class FunctionBaseTest : public testing::Test, // C++ value. Arguments should be referenced using c0, c1, .. cn. Supports // integers, floats, booleans, and strings. // + // Custom types such as TimestampWithTimezone are resolved and return + // their internal physical type (CustomType::type) using the + // `UnwrapCustomType` utility. + // // Example: // std::optional exp(std::optional a) { // return evaluateOnce("exp(c0)", a); @@ -166,7 +171,7 @@ class FunctionBaseTest : public testing::Test, // EXPECT_EQ(1, exp(0)); // EXPECT_EQ(std::nullopt, exp(std::nullopt)); template - std::optional evaluateOnce( + std::optional::type> evaluateOnce( const std::string& expr, const std::optional&... args) { return evaluateOnce( @@ -176,7 +181,7 @@ class FunctionBaseTest : public testing::Test, } template - std::optional evaluateOnce( + std::optional::type> evaluateOnce( const std::string& expr, const std::vector>& args, const std::vector& types, @@ -192,7 +197,7 @@ class FunctionBaseTest : public testing::Test, } template - std::optional evaluateOnce( + std::optional::type> evaluateOnce( const std::string& expr, const RowVectorPtr rowVectorPtr, const std::optional& rows = std::nullopt, @@ -200,8 +205,9 @@ class FunctionBaseTest : public testing::Test, auto result = evaluate>>( expr, rowVectorPtr, rows, resultType); - return result->isNullAt(0) ? std::optional{} - : ReturnType(result->valueAt(0)); + return result->isNullAt(0) + ? std::optional::type>{} + : typename UnwrapCustomType::type(result->valueAt(0)); } core::TypedExprPtr parseExpression(