diff --git a/.gitignore b/.gitignore index 9264fd73c..48d26fc09 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ ligradle .DS_Store *.patch */metastore_db -.pyc \ No newline at end of file +.pyc +__pycache__ diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/DataTypeDerivedSqlCallConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/DataTypeDerivedSqlCallConverter.java index 056120e91..f23a8ebf6 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/DataTypeDerivedSqlCallConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/DataTypeDerivedSqlCallConverter.java @@ -1,10 +1,13 @@ /** - * Copyright 2022-2023 LinkedIn Corporation. All rights reserved. + * Copyright 2022-2024 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ package com.linkedin.coral.trino.rel2trino; +import java.util.List; +import java.util.Map; + import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlNode; @@ -47,6 +50,17 @@ public DataTypeDerivedSqlCallConverter(HiveMetastoreClient mscClient, SqlNode to new UnionSqlCallTransformer(typeDerivationUtil)); } + public DataTypeDerivedSqlCallConverter(Map>> localMetaStore, SqlNode topSqlNode) { + toRelConverter = new HiveToRelConverter(localMetaStore); + topSqlNode.accept(new RegisterDynamicFunctionsForTypeDerivation()); + + TypeDerivationUtil typeDerivationUtil = new TypeDerivationUtil(toRelConverter.getSqlValidator(), topSqlNode); + operatorTransformerList = SqlCallTransformers.of(new FromUtcTimestampOperatorTransformer(typeDerivationUtil), + new GenericProjectTransformer(typeDerivationUtil), new NamedStructToCastTransformer(typeDerivationUtil), + new ConcatOperatorTransformer(typeDerivationUtil), new SubstrOperatorTransformer(typeDerivationUtil), + new UnionSqlCallTransformer(typeDerivationUtil)); + } + @Override public SqlNode visit(final SqlCall call) { return operatorTransformerList.apply((SqlCall) super.visit(call)); diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/RelToTrinoConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/RelToTrinoConverter.java index 416c0299f..7cff79033 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/RelToTrinoConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/RelToTrinoConverter.java @@ -1,5 +1,5 @@ /** - * Copyright 2017-2023 LinkedIn Corporation. All rights reserved. + * Copyright 2017-2024 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -56,6 +56,7 @@ public class RelToTrinoConverter extends RelToSqlConverter { */ private Map configs = new HashMap<>(); private HiveMetastoreClient _hiveMetastoreClient; + private Map>> _localMetaStore; /** * Creates a RelToTrinoConverter. @@ -78,6 +79,18 @@ public RelToTrinoConverter(HiveMetastoreClient mscClient, Map c _hiveMetastoreClient = mscClient; } + /** + * Creates a RelToTrinoConverter. + * @param localMetaStore In-memory Hive metastore represented in a map. + * @param configs configs + */ + public RelToTrinoConverter(Map>> localMetaStore, Map configs) { + super(TrinoSqlDialect.INSTANCE); + checkNotNull(configs); + this.configs = configs; + _localMetaStore = localMetaStore; + } + /** * Convert relational algebra to Trino's SQL * @param relNode calcite relational algebra representation of SQL @@ -87,8 +100,17 @@ public String convert(RelNode relNode) { RelNode rel = convertRel(relNode, configs); SqlNode sqlNode = convertToSqlNode(rel); - SqlNode sqlNodeWithRelDataTypeDerivedConversions = - sqlNode.accept(new DataTypeDerivedSqlCallConverter(_hiveMetastoreClient, sqlNode)); + DataTypeDerivedSqlCallConverter sqlNodeWithRelDataTypeDerivedConverter; + + if (_localMetaStore != null && _hiveMetastoreClient == null) { + sqlNodeWithRelDataTypeDerivedConverter = new DataTypeDerivedSqlCallConverter(_localMetaStore, sqlNode); + } else if (_hiveMetastoreClient != null && _localMetaStore == null) { + sqlNodeWithRelDataTypeDerivedConverter = new DataTypeDerivedSqlCallConverter(_hiveMetastoreClient, sqlNode); + } else { + throw new IllegalStateException("Metadata source must be provided"); + } + + SqlNode sqlNodeWithRelDataTypeDerivedConversions = sqlNode.accept(sqlNodeWithRelDataTypeDerivedConverter); SqlNode sqlNodeWithUDFOperatorConverted = sqlNodeWithRelDataTypeDerivedConversions.accept(new CoralToTrinoSqlCallConverter(configs));