diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java index 6e840fffc..f29fe4636 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java @@ -69,7 +69,7 @@ protected SqlRexConvertletTable getConvertletTable() { } @Override - protected SqlValidator getSqlValidator() { + public SqlValidator getSqlValidator() { return sqlValidator; } diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/Calcite2TrinoUDFConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/Calcite2TrinoUDFConverter.java index 7b34c90d6..918299707 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/Calcite2TrinoUDFConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/Calcite2TrinoUDFConverter.java @@ -5,7 +5,6 @@ */ package com.linkedin.coral.trino.rel2trino; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -28,7 +27,6 @@ import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.logical.LogicalUnion; import org.apache.calcite.rel.logical.LogicalValues; -import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -42,12 +40,10 @@ import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import com.linkedin.coral.com.google.common.collect.ImmutableList; -import com.linkedin.coral.common.functions.FunctionReturnTypes; import com.linkedin.coral.common.functions.GenericProjectFunction; import com.linkedin.coral.trino.rel2trino.functions.GenericProjectToTrinoConverter; import static com.linkedin.coral.trino.rel2trino.CoralTrinoConfigKeys.*; -import static org.apache.calcite.sql.fun.SqlStdOperatorTable.MULTIPLY; import static org.apache.calcite.sql.type.ReturnTypes.explicit; import static org.apache.calcite.sql.type.SqlTypeName.*; @@ -175,20 +171,6 @@ public RexNode visitCall(RexCall call) { final String operatorName = call.getOperator().getName(); - if (operatorName.equalsIgnoreCase("from_utc_timestamp")) { - Optional modifiedCall = visitFromUtcTimestampCall(call); - if (modifiedCall.isPresent()) { - return modifiedCall.get(); - } - } - - if (operatorName.equalsIgnoreCase("from_unixtime")) { - Optional modifiedCall = visitFromUnixtime(call); - if (modifiedCall.isPresent()) { - return modifiedCall.get(); - } - } - if (operatorName.equalsIgnoreCase("cast")) { Optional modifiedCall = visitCast(call); if (modifiedCall.isPresent()) { @@ -231,80 +213,6 @@ private Optional visitConcat(RexCall call) { return Optional.of(rexBuilder.makeCall(op, castOperands)); } - private Optional visitFromUnixtime(RexCall call) { - List convertedOperands = visitList(call.getOperands(), (boolean[]) null); - SqlOperator formatDatetime = createSqlOperatorOfFunction("format_datetime", FunctionReturnTypes.STRING); - SqlOperator fromUnixtime = createSqlOperatorOfFunction("from_unixtime", explicit(TIMESTAMP)); - if (convertedOperands.size() == 1) { - return Optional - .of(rexBuilder.makeCall(formatDatetime, rexBuilder.makeCall(fromUnixtime, call.getOperands().get(0)), - rexBuilder.makeLiteral("yyyy-MM-dd HH:mm:ss"))); - } else if (convertedOperands.size() == 2) { - return Optional.of(rexBuilder.makeCall(formatDatetime, - rexBuilder.makeCall(fromUnixtime, call.getOperands().get(0)), call.getOperands().get(1))); - } - return Optional.empty(); - } - - private Optional visitFromUtcTimestampCall(RexCall call) { - RelDataType inputType = call.getOperands().get(0).getType(); - // TODO(trinodb/trino#6295) support high-precision timestamp - RelDataType targetType = typeFactory.createSqlType(TIMESTAMP, 3); - - List convertedOperands = visitList(call.getOperands(), (boolean[]) null); - RexNode sourceValue = convertedOperands.get(0); - RexNode timezone = convertedOperands.get(1); - - // In below definitions we should use `TIMESTATMP WITH TIME ZONE`. As calcite is lacking - // this type we use `TIMESTAMP` instead. It does not have any practical implications as result syntax tree - // is not type-checked, and only used for generating output SQL for a view query. - SqlOperator trinoAtTimeZone = - createSqlOperatorOfFunction("at_timezone", explicit(TIMESTAMP /* should be WITH TIME ZONE */)); - SqlOperator trinoWithTimeZone = - createSqlOperatorOfFunction("with_timezone", explicit(TIMESTAMP /* should be WITH TIME ZONE */)); - SqlOperator trinoToUnixTime = createSqlOperatorOfFunction("to_unixtime", explicit(DOUBLE)); - SqlOperator trinoFromUnixtimeNanos = - createSqlOperatorOfFunction("from_unixtime_nanos", explicit(TIMESTAMP /* should be WITH TIME ZONE */)); - SqlOperator trinoFromUnixTime = - createSqlOperatorOfFunction("from_unixtime", explicit(TIMESTAMP /* should be WITH TIME ZONE */)); - SqlOperator trinoCanonicalizeHiveTimezoneId = - createSqlOperatorOfFunction("$canonicalize_hive_timezone_id", explicit(VARCHAR)); - - RelDataType bigintType = typeFactory.createSqlType(BIGINT); - RelDataType doubleType = typeFactory.createSqlType(DOUBLE); - - if (inputType.getSqlTypeName() == BIGINT || inputType.getSqlTypeName() == INTEGER - || inputType.getSqlTypeName() == SMALLINT || inputType.getSqlTypeName() == TINYINT) { - - return Optional.of(rexBuilder.makeCast(targetType, - rexBuilder.makeCall(trinoAtTimeZone, - rexBuilder.makeCall(trinoFromUnixtimeNanos, - rexBuilder.makeCall(MULTIPLY, rexBuilder.makeCast(bigintType, sourceValue), - rexBuilder.makeBigintLiteral(BigDecimal.valueOf(1000000)))), - rexBuilder.makeCall(trinoCanonicalizeHiveTimezoneId, timezone)))); - } - - if (inputType.getSqlTypeName() == DOUBLE || inputType.getSqlTypeName() == FLOAT - || inputType.getSqlTypeName() == DECIMAL) { - - return Optional.of(rexBuilder.makeCast(targetType, - rexBuilder.makeCall(trinoAtTimeZone, - rexBuilder.makeCall(trinoFromUnixTime, rexBuilder.makeCast(doubleType, sourceValue)), - rexBuilder.makeCall(trinoCanonicalizeHiveTimezoneId, timezone)))); - } - - if (inputType.getSqlTypeName() == TIMESTAMP || inputType.getSqlTypeName() == DATE) { - return Optional.of(rexBuilder.makeCast(targetType, - rexBuilder.makeCall(trinoAtTimeZone, - rexBuilder.makeCall(trinoFromUnixTime, - rexBuilder.makeCall(trinoToUnixTime, - rexBuilder.makeCall(trinoWithTimeZone, sourceValue, rexBuilder.makeLiteral("UTC")))), - rexBuilder.makeCall(trinoCanonicalizeHiveTimezoneId, timezone)))); - } - - return Optional.empty(); - } - // Hive allows passing in a byte array or String to substr/substring, so we can make an effort to emulate the // behavior by casting non-String input to String // https://cwiki.apache.org/confluence/display/hive/languagemanual+udf diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/CoralToTrinoSqlCallConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/CoralToTrinoSqlCallConverter.java index 4ce175dc9..142360ff4 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/CoralToTrinoSqlCallConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/CoralToTrinoSqlCallConverter.java @@ -27,6 +27,7 @@ import com.linkedin.coral.trino.rel2trino.transformers.CollectListOrSetFunctionTransformer; import com.linkedin.coral.trino.rel2trino.transformers.CoralRegistryOperatorRenameSqlCallTransformer; import com.linkedin.coral.trino.rel2trino.transformers.CurrentTimestampTransformer; +import com.linkedin.coral.trino.rel2trino.transformers.FromUnixtimeOperatorTransformer; import com.linkedin.coral.trino.rel2trino.transformers.GenericCoralRegistryOperatorRenameSqlCallTransformer; import com.linkedin.coral.trino.rel2trino.transformers.JoinSqlCallTransformer; import com.linkedin.coral.trino.rel2trino.transformers.MapValueConstructorTransformer; @@ -104,7 +105,7 @@ protected SqlCall transform(SqlCall sqlCall) { + "{\"op\": \"date\", \"operands\":[{\"op\": \"timestamp\", \"operands\":[{\"input\": 1}]}]}]", null, null), new ToDateOperatorTransformer(configs.getOrDefault(AVOID_TRANSFORM_TO_DATE_UDF, false)), - new CurrentTimestampTransformer(), + new CurrentTimestampTransformer(), new FromUnixtimeOperatorTransformer(), // LinkedIn specific functions new CoralRegistryOperatorRenameSqlCallTransformer( diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/DataTypeDerivedSqlCallConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/DataTypeDerivedSqlCallConverter.java new file mode 100644 index 000000000..8db5cca72 --- /dev/null +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/DataTypeDerivedSqlCallConverter.java @@ -0,0 +1,41 @@ +/** + * Copyright 2022-2023 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.coral.trino.rel2trino; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.calcite.sql.validate.SqlValidator; + +import com.linkedin.coral.common.HiveMetastoreClient; +import com.linkedin.coral.common.transformers.SqlCallTransformers; +import com.linkedin.coral.common.utils.TypeDerivationUtil; +import com.linkedin.coral.hive.hive2rel.HiveToRelConverter; +import com.linkedin.coral.trino.rel2trino.transformers.FromUtcTimestampOperatorTransformer; + + +/** + * DataTypeDerivedSqlCallConverter transforms the sqlCalls + * in the input SqlNode representation to be compatible with Trino engine. + * The transformation may involve change in operator, reordering the operands + * or even re-constructing the SqlNode. + * + * All the transformations performed as part of this shuttle require RelDataType derivation. + */ +public class DataTypeDerivedSqlCallConverter extends SqlShuttle { + private final SqlCallTransformers operatorTransformerList; + + public DataTypeDerivedSqlCallConverter(HiveMetastoreClient mscClient, SqlNode topSqlNode) { + SqlValidator sqlValidator = new HiveToRelConverter(mscClient).getSqlValidator(); + TypeDerivationUtil typeDerivationUtil = new TypeDerivationUtil(sqlValidator, topSqlNode); + operatorTransformerList = SqlCallTransformers.of(new FromUtcTimestampOperatorTransformer(typeDerivationUtil)); + } + + @Override + public SqlNode visit(final SqlCall call) { + return operatorTransformerList.apply((SqlCall) super.visit(call)); + } +} diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/RelToTrinoConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/RelToTrinoConverter.java index 80e4fd705..47b87a6e8 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/RelToTrinoConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/RelToTrinoConverter.java @@ -37,8 +37,6 @@ import static com.google.common.base.Preconditions.*; import static com.linkedin.coral.trino.rel2trino.Calcite2TrinoUDFConverter.convertRel; import static com.linkedin.coral.trino.rel2trino.CoralTrinoConfigKeys.*; -import static org.apache.calcite.sql.fun.SqlStdOperatorTable.*; -import static org.apache.calcite.sql.parser.SqlParserPos.*; public class RelToTrinoConverter extends RelToSqlConverter { @@ -88,7 +86,12 @@ public RelToTrinoConverter(HiveMetastoreClient mscClient, Map c public String convert(RelNode relNode) { RelNode rel = convertRel(relNode, configs); SqlNode sqlNode = convertToSqlNode(rel); - SqlNode sqlNodeWithUDFOperatorConverted = sqlNode.accept(new CoralToTrinoSqlCallConverter(configs)); + + SqlNode sqlNodeWithRelDataTypeDerivedConversions = + sqlNode.accept(new DataTypeDerivedSqlCallConverter(_hiveMetastoreClient, sqlNode)); + + SqlNode sqlNodeWithUDFOperatorConverted = + sqlNodeWithRelDataTypeDerivedConversions.accept(new CoralToTrinoSqlCallConverter(configs)); return sqlNodeWithUDFOperatorConverted.accept(new TrinoSqlRewriter()).toSqlString(TrinoSqlDialect.INSTANCE) .toString(); } diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/FromUnixtimeOperatorTransformer.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/FromUnixtimeOperatorTransformer.java new file mode 100644 index 000000000..59eb09159 --- /dev/null +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/FromUnixtimeOperatorTransformer.java @@ -0,0 +1,56 @@ +/** + * Copyright 2023 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.coral.trino.rel2trino.transformers; + +import java.util.List; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.parser.SqlParserPos; + +import com.linkedin.coral.common.functions.FunctionReturnTypes; +import com.linkedin.coral.common.transformers.SqlCallTransformer; + +import static org.apache.calcite.sql.type.ReturnTypes.*; +import static org.apache.calcite.sql.type.SqlTypeName.*; + + +/** + * This transformer operates on SqlCalls with 'FROM_UNIXTIME(x)' Coral IR function + * and transforms it to trino engine compatible function - FORMAT_DATETIME(FROM_UNIXTIME(x)). + * For Example: + * A SqlCall of the form: "FROM_UNIXTIME(10000)" is transformed to + * "FORMAT_DATETIME(FROM_UNIXTIME(10000), 'yyyy-MM-dd HH:mm:ss')" + */ +public class FromUnixtimeOperatorTransformer extends SqlCallTransformer { + + private static final String FORMAT_DATETIME = "format_datetime"; + private static final String FROM_UNIXTIME = "from_unixtime"; + + @Override + protected boolean condition(SqlCall sqlCall) { + return sqlCall.getOperator().getName().equalsIgnoreCase(FROM_UNIXTIME); + } + + @Override + protected SqlCall transform(SqlCall sqlCall) { + SqlOperator formatDatetimeOperator = createSqlOperator(FORMAT_DATETIME, FunctionReturnTypes.STRING); + SqlOperator fromUnixtimeOperator = createSqlOperator(FROM_UNIXTIME, explicit(TIMESTAMP)); + + List operands = sqlCall.getOperandList(); + if (operands.size() == 1) { + return formatDatetimeOperator.createCall(SqlParserPos.ZERO, + fromUnixtimeOperator.createCall(SqlParserPos.ZERO, operands.get(0)), + SqlLiteral.createCharString("yyyy-MM-dd HH:mm:ss", SqlParserPos.ZERO)); + } else if (operands.size() == 2) { + return formatDatetimeOperator.createCall(SqlParserPos.ZERO, + fromUnixtimeOperator.createCall(SqlParserPos.ZERO, operands.get(0)), operands.get(1)); + } + return sqlCall; + } +} diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/FromUtcTimestampOperatorTransformer.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/FromUtcTimestampOperatorTransformer.java new file mode 100644 index 000000000..7d7535cd1 --- /dev/null +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/FromUtcTimestampOperatorTransformer.java @@ -0,0 +1,133 @@ +/** + * Copyright 2023 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.coral.trino.rel2trino.transformers; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlBasicTypeNameSpec; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.BasicSqlType; +import org.apache.calcite.sql.type.SqlTypeName; + +import com.linkedin.coral.common.HiveTypeSystem; +import com.linkedin.coral.common.calcite.CalciteUtil; +import com.linkedin.coral.common.transformers.SqlCallTransformer; +import com.linkedin.coral.common.utils.TypeDerivationUtil; + +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.*; +import static org.apache.calcite.sql.parser.SqlParserPos.*; +import static org.apache.calcite.sql.type.ReturnTypes.*; +import static org.apache.calcite.sql.type.SqlTypeName.*; +import static org.apache.calcite.sql.type.SqlTypeName.BIGINT; +import static org.apache.calcite.sql.type.SqlTypeName.DATE; +import static org.apache.calcite.sql.type.SqlTypeName.DOUBLE; +import static org.apache.calcite.sql.type.SqlTypeName.INTEGER; + + +/** + * This transformer operates on SqlCalls with 'FROM_UTC_TIMESTAMP(x, timezone)' Coral IR function + * and transforms it into different trino engine compatible functions based on the data type of 'x'. + * For Example: + * A SqlCall of the form: "FROM_UTC_TIMESTAMP(10000, 'America/Los_Angeles')" is transformed to + * "CAST(AT_TIMEZONE(FROM_UNIXTIME_NANOS(CAST(10000 AS BIGINT) * 1000000), $CANONICALIZE_HIVE_TIMEZONE_ID('America/Los_Angeles')) AS TIMESTAMP(3))" + */ +public class FromUtcTimestampOperatorTransformer extends SqlCallTransformer { + + private static final Set INT_SQL_TYPE_NAMES = + new HashSet<>(Arrays.asList(BIGINT, INTEGER, SMALLINT, TINYINT)); + private static final Set DECIMAL_SQL_TYPE_NAMES = new HashSet<>(Arrays.asList(DOUBLE, FLOAT, DECIMAL)); + private static final Set TIMESTAMP_SQL_TYPE_NAMES = new HashSet<>(Arrays.asList(TIMESTAMP, DATE)); + private static final String FROM_UTC_TIMESTAMP = "from_utc_timestamp"; + private static final String AT_TIMEZONE = "at_timezone"; + private static final String WITH_TIMEZONE = "with_timezone"; + private static final String TO_UNIXTIME = "to_unixtime"; + private static final String FROM_UNIXTIME_NANOS = "from_unixtime_nanos"; + private static final String FROM_UNIXTIME = "from_unixtime"; + private static final String CANONICALIZE_HIVE_TIMEZONE_ID = "$canonicalize_hive_timezone_id"; + + public FromUtcTimestampOperatorTransformer(TypeDerivationUtil typeDerivationUtil) { + super(typeDerivationUtil); + } + + @Override + protected boolean condition(SqlCall sqlCall) { + return sqlCall.getOperator().getName().equalsIgnoreCase(FROM_UTC_TIMESTAMP); + } + + @Override + protected SqlCall transform(SqlCall sqlCall) { + List operands = sqlCall.getOperandList(); + SqlNode sourceValue = operands.get(0); + SqlNode timezone = operands.get(1); + + RelDataType targetType = new BasicSqlType(new HiveTypeSystem(), TIMESTAMP, 3); + RelDataType inputType = deriveRelDatatype(sqlCall.getOperandList().get(0)); + SqlTypeName inputSqlTypeName = inputType.getSqlTypeName(); + + // In below definitions we should use `TIMESTATMP WITH TIME ZONE`. As calcite is lacking + // this type we use `TIMESTAMP` instead. It does not have any practical implications as result syntax tree + // is not type-checked, and only used for generating output SQL for a view query. + SqlOperator trinoAtTimeZone = createSqlOperator(AT_TIMEZONE, explicit(TIMESTAMP /* should be WITH TIME ZONE */)); + SqlOperator trinoWithTimeZone = + createSqlOperator(WITH_TIMEZONE, explicit(TIMESTAMP /* should be WITH TIME ZONE */)); + SqlOperator trinoToUnixTime = createSqlOperator(TO_UNIXTIME, explicit(DOUBLE)); + SqlOperator trinoFromUnixtimeNanos = + createSqlOperator(FROM_UNIXTIME_NANOS, explicit(TIMESTAMP /* should be WITH TIME ZONE */)); + SqlOperator trinoFromUnixTime = + createSqlOperator(FROM_UNIXTIME, explicit(TIMESTAMP /* should be WITH TIME ZONE */)); + SqlOperator trinoCanonicalizeHiveTimezoneId = createSqlOperator(CANONICALIZE_HIVE_TIMEZONE_ID, explicit(VARCHAR)); + + SqlCall canonicalizeHiveTimezoneIdSqlCall = trinoCanonicalizeHiveTimezoneId.createCall(ZERO, timezone); + + if (INT_SQL_TYPE_NAMES.contains(inputSqlTypeName)) { + + SqlCall castedOperandCall = castOperand(sourceValue, new BasicSqlType(new HiveTypeSystem(), BIGINT)); + SqlCall multipliedValueCall = + MULTIPLY.createCall(ZERO, castedOperandCall, CalciteUtil.createLiteralNumber(1000000, ZERO)); + SqlCall fromUnixtimeNanosCall = trinoFromUnixtimeNanos.createCall(ZERO, multipliedValueCall); + SqlCall atTimeZoneCall = + trinoAtTimeZone.createCall(ZERO, fromUnixtimeNanosCall, canonicalizeHiveTimezoneIdSqlCall); + + return castOperand(atTimeZoneCall, targetType); + } else if (DECIMAL_SQL_TYPE_NAMES.contains(inputSqlTypeName)) { + + SqlCall castedOperandCall = castOperand(sourceValue, new BasicSqlType(new HiveTypeSystem(), DOUBLE)); + SqlCall fromUnixTimeCall = trinoFromUnixTime.createCall(ZERO, castedOperandCall); + SqlCall atTimeZoneCall = trinoAtTimeZone.createCall(ZERO, fromUnixTimeCall, canonicalizeHiveTimezoneIdSqlCall); + + return castOperand(atTimeZoneCall, targetType); + } else if (TIMESTAMP_SQL_TYPE_NAMES.contains(inputSqlTypeName)) { + + SqlCall withTimeZoneCall = + trinoWithTimeZone.createCall(ZERO, sourceValue, CalciteUtil.createStringLiteral("UTC", ZERO)); + SqlCall toUnixTimeCall = trinoToUnixTime.createCall(ZERO, withTimeZoneCall); + SqlCall fromUnixTimeCall = trinoFromUnixTime.createCall(ZERO, toUnixTimeCall); + SqlCall atTimeZoneCall = trinoAtTimeZone.createCall(ZERO, fromUnixTimeCall, canonicalizeHiveTimezoneIdSqlCall); + + return castOperand(atTimeZoneCall, targetType); + } + return sqlCall; + } + + private SqlCall castOperand(SqlNode operand, RelDataType relDataType) { + return SqlStdOperatorTable.CAST.createCall(ZERO, operand, getSqlDataTypeSpecForCasting(relDataType)); + } + + private SqlDataTypeSpec getSqlDataTypeSpecForCasting(RelDataType relDataType) { + final SqlTypeNameSpec typeNameSpec = new SqlBasicTypeNameSpec(relDataType.getSqlTypeName(), + relDataType.getPrecision(), relDataType.getScale(), null, ZERO); + return new SqlDataTypeSpec(typeNameSpec, ZERO); + } +} diff --git a/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/HiveToTrinoConverterTest.java b/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/HiveToTrinoConverterTest.java index 3fa2b4fd5..6c90a42a0 100644 --- a/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/HiveToTrinoConverterTest.java +++ b/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/HiveToTrinoConverterTest.java @@ -159,7 +159,7 @@ public Object[][] viewTestCasesProvider() { { "test", "get_json_object_view", "SELECT \"json_extract\"(\"tablea\".\"b\".\"b1\", '$.name')\n" + "FROM \"test\".\"tablea\" AS \"tablea\"" }, - { "test", "view_from_utc_timestamp", "SELECT CAST(\"at_timezone\"(\"from_unixtime_nanos\"(CAST(\"table_from_utc_timestamp\".\"a_tinyint\" AS BIGINT) * 1000000), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime_nanos\"(CAST(\"table_from_utc_timestamp\".\"a_smallint\" AS BIGINT) * 1000000), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime_nanos\"(CAST(\"table_from_utc_timestamp\".\"a_integer\" AS BIGINT) * 1000000), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime_nanos\"(CAST(\"table_from_utc_timestamp\".\"a_bigint\" AS BIGINT) * 1000000), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime\"(CAST(\"table_from_utc_timestamp\".\"a_float\" AS DOUBLE)), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime\"(CAST(\"table_from_utc_timestamp\".\"a_double\" AS DOUBLE)), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime\"(CAST(\"table_from_utc_timestamp\".\"a_decimal_three\" AS DOUBLE)), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime\"(CAST(\"table_from_utc_timestamp\".\"a_decimal_zero\" AS DOUBLE)), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime\"(\"to_unixtime\"(\"with_timezone\"(\"table_from_utc_timestamp\".\"a_timestamp\", 'UTC'))), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime\"(\"to_unixtime\"(\"with_timezone\"(\"table_from_utc_timestamp\".\"a_date\", 'UTC'))), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3))\n" + { "test", "view_from_utc_timestamp", "SELECT CAST(\"at_timezone\"(\"from_unixtime_nanos\"(CAST(\"table_from_utc_timestamp\".\"a_tinyint\" AS BIGINT) * 1000000), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime_nanos\"(CAST(\"table_from_utc_timestamp\".\"a_smallint\" AS BIGINT) * 1000000), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime_nanos\"(CAST(\"table_from_utc_timestamp\".\"a_integer\" AS BIGINT) * 1000000), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"from_unixtime_nanos\"(CAST(\"table_from_utc_timestamp\".\"a_bigint\" AS BIGINT) * 1000000), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"format_datetime\"(\"from_unixtime\"(CAST(\"table_from_utc_timestamp\".\"a_float\" AS DOUBLE)), 'yyyy-MM-dd HH:mm:ss'), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"format_datetime\"(\"from_unixtime\"(CAST(\"table_from_utc_timestamp\".\"a_double\" AS DOUBLE)), 'yyyy-MM-dd HH:mm:ss'), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"format_datetime\"(\"from_unixtime\"(CAST(\"table_from_utc_timestamp\".\"a_decimal_three\" AS DOUBLE)), 'yyyy-MM-dd HH:mm:ss'), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"format_datetime\"(\"from_unixtime\"(CAST(\"table_from_utc_timestamp\".\"a_decimal_zero\" AS DOUBLE)), 'yyyy-MM-dd HH:mm:ss'), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"format_datetime\"(\"from_unixtime\"(\"to_unixtime\"(\"with_timezone\"(\"table_from_utc_timestamp\".\"a_timestamp\", 'UTC'))), 'yyyy-MM-dd HH:mm:ss'), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), CAST(\"at_timezone\"(\"format_datetime\"(\"from_unixtime\"(\"to_unixtime\"(\"with_timezone\"(\"table_from_utc_timestamp\".\"a_date\", 'UTC'))), 'yyyy-MM-dd HH:mm:ss'), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3))\n" + "FROM \"test\".\"table_from_utc_timestamp\" AS \"table_from_utc_timestamp\"" }, { "test", "date_calculation_view", "SELECT \"date\"(CAST(\"substr\"('2021-08-20', 1, 10) AS TIMESTAMP)), \"date\"(CAST('2021-08-20' AS TIMESTAMP)), \"date\"(CAST('2021-08-20 00:00:00' AS TIMESTAMP)), \"date_add\"('day', 1, \"date\"(CAST('2021-08-20' AS TIMESTAMP))), \"date_add\"('day', 1, \"date\"(CAST('2021-08-20 00:00:00' AS TIMESTAMP))), \"date_add\"('day', 1 * -1, \"date\"(CAST('2021-08-20' AS TIMESTAMP))), \"date_add\"('day', 1 * -1, \"date\"(CAST('2021-08-20 00:00:00' AS TIMESTAMP))), CAST(\"date_diff\"('day', \"date\"(CAST('2021-08-21' AS TIMESTAMP)), \"date\"(CAST('2021-08-20' AS TIMESTAMP))) AS INTEGER), CAST(\"date_diff\"('day', \"date\"(CAST('2021-08-19' AS TIMESTAMP)), \"date\"(CAST('2021-08-20' AS TIMESTAMP))) AS INTEGER), CAST(\"date_diff\"('day', \"date\"(CAST('2021-08-19 23:59:59' AS TIMESTAMP)), \"date\"(CAST('2021-08-20 00:00:00' AS TIMESTAMP))) AS INTEGER)\n" @@ -467,7 +467,7 @@ public void testCastNestedTimestampToDecimal() { relNode = TestUtils.getHiveToRelConverter().convertSql( "SELECT CAST(from_utc_timestamp(a_date, 'America/Los_Angeles') AS DECIMAL(10, 0)) AS d\nFROM test.table_from_utc_timestamp"); targetSql = - "SELECT CAST(\"to_unixtime\"(\"with_timezone\"(CAST(\"at_timezone\"(\"from_unixtime\"(\"to_unixtime\"(\"with_timezone\"(\"table_from_utc_timestamp0\".\"a_date\", 'UTC'))), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), 'UTC')) AS DECIMAL(10, 0)) AS \"d\"\n" + "SELECT CAST(\"to_unixtime\"(\"with_timezone\"(CAST(\"at_timezone\"(\"format_datetime\"(\"from_unixtime\"(\"to_unixtime\"(\"with_timezone\"(\"table_from_utc_timestamp0\".\"a_date\", 'UTC'))), 'yyyy-MM-dd HH:mm:ss'), \"$canonicalize_hive_timezone_id\"('America/Los_Angeles')) AS TIMESTAMP(3)), 'UTC')) AS DECIMAL(10, 0)) AS \"d\"\n" + "FROM \"test\".\"table_from_utc_timestamp\" AS \"table_from_utc_timestamp0\""; expandedSql = relToTrinoConverter.convert(relNode); assertEquals(expandedSql, targetSql);