From a8b86e71847971a420161f566b1d496421926033 Mon Sep 17 00:00:00 2001 From: Jiefan Li Date: Tue, 27 Aug 2024 19:33:31 -0400 Subject: [PATCH] [Coral-Hive] Extend VersionedSqlUserDefinedFunction to support version-specific function names (#508) --- ...SourceOperatorMatchSqlCallTransformer.java | 4 +- .../functions/HiveFunctionResolver.java | 92 ++++++++++++------- .../VersionedSqlUserDefinedFunction.java | 76 +++++++++++---- .../hive/hive2rel/HiveToRelConverterTest.java | 8 ++ .../coral/hive/hive2rel/TestUtils.java | 12 +++ .../coral/hive/hive2rel/CoralTestUDF.java | 17 ++++ .../pig/rel2pig/rel/functions/PigUDF.java | 4 +- .../transformers/HiveUDFTransformer.java | 8 +- .../transformers/TransportUDFTransformer.java | 8 +- .../linkedin/coral/spark/CoralSparkTest.java | 34 +++++++ .../com/linkedin/coral/spark/TestUtils.java | 8 ++ .../hive/hive2rel/CoralTestVersionedUDF.java | 17 ++++ .../CoralToTrinoSqlCallConverter.java | 28 +----- ...istryOperatorRenameSqlCallTransformer.java | 2 +- ...istryOperatorRenameSqlCallTransformer.java | 44 --------- .../transformers/HiveUDFTransformer.java | 39 ++++++++ .../rel2trino/HiveToTrinoConverterTest.java | 17 ++++ .../coral/trino/rel2trino/TestUtils.java | 8 ++ .../coral/hive/hive2rel/CoralTestUDF.java | 17 ++++ 19 files changed, 308 insertions(+), 135 deletions(-) create mode 100644 coral-hive/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestUDF.java create mode 100644 coral-spark/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestVersionedUDF.java delete mode 100644 coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/GenericCoralRegistryOperatorRenameSqlCallTransformer.java create mode 100644 coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/HiveUDFTransformer.java create mode 100644 coral-trino/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestUDF.java diff --git a/coral-common/src/main/java/com/linkedin/coral/common/transformers/SourceOperatorMatchSqlCallTransformer.java b/coral-common/src/main/java/com/linkedin/coral/common/transformers/SourceOperatorMatchSqlCallTransformer.java index d924a7087..fe3356766 100644 --- a/coral-common/src/main/java/com/linkedin/coral/common/transformers/SourceOperatorMatchSqlCallTransformer.java +++ b/coral-common/src/main/java/com/linkedin/coral/common/transformers/SourceOperatorMatchSqlCallTransformer.java @@ -1,5 +1,5 @@ /** - * Copyright 2023 LinkedIn Corporation. All rights reserved. + * Copyright 2023-2024 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -7,8 +7,6 @@ import org.apache.calcite.sql.SqlCall; -import static com.linkedin.coral.common.calcite.CalciteUtil.*; - /** * This class is a subclass of {@link SqlCallTransformer} which transforms a function operator on SqlNode layer diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java index 317287446..4683f3059 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java @@ -1,5 +1,5 @@ /** - * Copyright 2018-2023 LinkedIn Corporation. All rights reserved. + * Copyright 2018-2024 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -44,6 +44,7 @@ * Class to resolve hive function names in SQL to Function. */ public class HiveFunctionResolver { + private static final String VERSIONED_UDF_CLASS_NAME_PREFIX = "coral_udf_version_(\\d+|x)_(\\d+|x)_(\\d+|x)"; public final FunctionRegistry registry; private final ConcurrentHashMap dynamicFunctionRegistry; @@ -111,7 +112,7 @@ public SqlOperator resolveBinaryOperator(String name) { * this attempts to match dali-style function names (DB_TABLE_VERSION_FUNCTION). * Right now, this method does not validate parameters leaving it to * the subsequent validator and analyzer phases to validate parameter types. - * @param functionName hive function name + * @param originalViewTextFunctionName original function name in view text to resolve * @param hiveTable handle to Hive table representing metastore information. This is used for resolving * Dali function names, which are resolved using table parameters * @param numOfOperands number of operands this function takes. This is needed to @@ -119,14 +120,15 @@ public SqlOperator resolveBinaryOperator(String name) { * @return resolved hive functions * @throws UnknownSqlFunctionException if the function name can not be resolved. */ - public Function tryResolve(@Nonnull String functionName, @Nullable Table hiveTable, int numOfOperands) { - checkNotNull(functionName); - Collection functions = registry.lookup(functionName); + public Function tryResolve(@Nonnull String originalViewTextFunctionName, @Nullable Table hiveTable, + int numOfOperands) { + checkNotNull(originalViewTextFunctionName); + Collection functions = registry.lookup(originalViewTextFunctionName); if (functions.isEmpty() && hiveTable != null) { - functions = tryResolveAsDaliFunction(functionName, hiveTable, numOfOperands); + functions = tryResolveAsDaliFunction(originalViewTextFunctionName, hiveTable, numOfOperands); } if (functions.isEmpty()) { - throw new UnknownSqlFunctionException(functionName); + throw new UnknownSqlFunctionException(originalViewTextFunctionName); } if (functions.size() == 1) { return functions.iterator().next(); @@ -160,7 +162,7 @@ public Collection resolve(String functionName) { /** * Tries to resolve function name as Dali function name using the provided Hive table catalog information. * This uses table parameters 'function' property to resolve the function name to the implementing class. - * @param functionName function name to resolve + * @param originalViewTextFunctionName original function name in view text to resolve * @param table Hive metastore table handle * @param numOfOperands number of operands this function takes. This is needed to * create SqlOperandTypeChecker to resolve Dali function dynamically @@ -168,58 +170,62 @@ public Collection resolve(String functionName) { * of `databaseName_tableName_udfName` or `udfName` (without `databaseName_tableName_` prefix) * @throws UnknownSqlFunctionException if the function name is in Dali function name format but there is no mapping */ - public Collection tryResolveAsDaliFunction(String functionName, @Nonnull Table table, int numOfOperands) { + public Collection tryResolveAsDaliFunction(String originalViewTextFunctionName, @Nonnull Table table, + int numOfOperands) { Preconditions.checkNotNull(table); String functionPrefix = String.format("%s_%s_", table.getDbName(), table.getTableName()); - if (!functionName.toLowerCase().startsWith(functionPrefix.toLowerCase())) { - // if functionName is not in `databaseName_tableName_udfName` format, we don't require the `databaseName_tableName_` prefix + if (!originalViewTextFunctionName.toLowerCase().startsWith(functionPrefix.toLowerCase())) { + // if originalViewTextFunctionName is not in `databaseName_tableName_udfName` format, we don't require the `databaseName_tableName_` prefix functionPrefix = ""; } - String funcBaseName = functionName.substring(functionPrefix.length()); + String funcBaseName = originalViewTextFunctionName.substring(functionPrefix.length()); HiveTable hiveTable = new HiveTable(table); Map functionParams = hiveTable.getDaliFunctionParams(); - String funcClassName = functionParams.get(funcBaseName); - if (funcClassName == null) { + String functionClassName = functionParams.get(funcBaseName); + if (functionClassName == null) { return ImmutableList.of(); } - final Collection Functions = registry.lookup(funcClassName); - if (Functions.size() == 0) { + // If the UDF class name is versioned, remove the versioning prefix, which allows user to + // register the unversioned UDF once and use different versioning prefix in the view + final Collection functions = registry.lookup(removeVersioningPrefix(functionClassName)); + if (functions.isEmpty()) { Collection dynamicResolvedFunctions = - resolveDaliFunctionDynamically(functionName, funcClassName, hiveTable, numOfOperands); + resolveDaliFunctionDynamically(originalViewTextFunctionName, functionClassName, hiveTable, numOfOperands); - if (dynamicResolvedFunctions.size() == 0) { + if (dynamicResolvedFunctions.isEmpty()) { // we want to see class name in the exception message for coverage testing // so throw exception here - throw new UnknownSqlFunctionException(funcClassName); + throw new UnknownSqlFunctionException(functionClassName); } return dynamicResolvedFunctions; } - return Functions.stream() - .map(f -> new Function(f.getFunctionName(), new VersionedSqlUserDefinedFunction( - (SqlUserDefinedFunction) f.getSqlOperator(), hiveTable.getDaliUdfDependencies(), functionName))) + return functions.stream() + .map(f -> new Function(f.getFunctionName(), + new VersionedSqlUserDefinedFunction((SqlUserDefinedFunction) f.getSqlOperator(), + hiveTable.getDaliUdfDependencies(), originalViewTextFunctionName, functionClassName))) .collect(Collectors.toList()); } - public void addDynamicFunctionToTheRegistry(String funcClassName, Function function) { - if (!dynamicFunctionRegistry.contains(funcClassName)) { - dynamicFunctionRegistry.put(funcClassName, function); + public void addDynamicFunctionToTheRegistry(String functionClassName, Function function) { + if (!dynamicFunctionRegistry.contains(functionClassName)) { + dynamicFunctionRegistry.put(functionClassName, function); } } - private @Nonnull Collection resolveDaliFunctionDynamically(String functionName, String funcClassName, - HiveTable hiveTable, int numOfOperands) { - if (dynamicFunctionRegistry.contains(funcClassName)) { - return ImmutableList.of(dynamicFunctionRegistry.get(functionName)); + private @Nonnull Collection resolveDaliFunctionDynamically(String originalViewTextFunctionName, + String functionClassName, HiveTable hiveTable, int numOfOperands) { + if (dynamicFunctionRegistry.contains(functionClassName)) { + return ImmutableList.of(dynamicFunctionRegistry.get(originalViewTextFunctionName)); } - Function function = new Function(funcClassName, + Function function = new Function(functionClassName, new VersionedSqlUserDefinedFunction( - new SqlUserDefinedFunction(new SqlIdentifier(funcClassName, ZERO), - new HiveGenericUDFReturnTypeInference(funcClassName, hiveTable.getDaliUdfDependencies()), null, + new SqlUserDefinedFunction(new SqlIdentifier(functionClassName, ZERO), + new HiveGenericUDFReturnTypeInference(functionClassName, hiveTable.getDaliUdfDependencies()), null, createSqlOperandTypeChecker(numOfOperands), null, null), - hiveTable.getDaliUdfDependencies(), functionName)); - dynamicFunctionRegistry.put(funcClassName, function); + hiveTable.getDaliUdfDependencies(), originalViewTextFunctionName, functionClassName)); + dynamicFunctionRegistry.put(functionClassName, function); return ImmutableList.of(function); } @@ -238,4 +244,22 @@ public void addDynamicFunctionToTheRegistry(String funcClassName, Function funct return sqlOperandTypeChecker; } + + /** + * Removes the versioning prefix from a given UDF class name if it is present. + * A class name is considered versioned if the prefix before the first dot + * follows {@link HiveFunctionResolver#VERSIONED_UDF_CLASS_NAME_PREFIX} format + */ + private String removeVersioningPrefix(String className) { + if (className != null && !className.isEmpty()) { + int firstDotIndex = className.indexOf('.'); + if (firstDotIndex != -1) { + String prefix = className.substring(0, firstDotIndex); + if (prefix.matches(VERSIONED_UDF_CLASS_NAME_PREFIX)) { + return className.substring(firstDotIndex + 1); + } + } + } + return className; + } } diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/VersionedSqlUserDefinedFunction.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/VersionedSqlUserDefinedFunction.java index 48d96fb10..0d962736a 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/VersionedSqlUserDefinedFunction.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/VersionedSqlUserDefinedFunction.java @@ -1,13 +1,15 @@ /** - * Copyright 2019-2022 LinkedIn Corporation. All rights reserved. + * Copyright 2019-2024 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ package com.linkedin.coral.hive.hive2rel.functions; import java.util.List; +import java.util.Map; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.schema.Function; @@ -23,61 +25,101 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; +import com.linkedin.coral.com.google.common.base.CaseFormat; +import com.linkedin.coral.com.google.common.base.Converter; + /** * Class that represents Dali versioned UDFs */ public class VersionedSqlUserDefinedFunction extends SqlUserDefinedFunction { + // Predefined map that associates class names with their corresponding short function names. + private static final Map SHORT_FUNC_NAME_MAP = ImmutableMap. builder() + .put("com.linkedin.dali.udf.watbotcrawlerlookup.hive.WATBotCrawlerLookup", "wat_bot_crawler_lookup") + .put("com.linkedin.stdudfs.parsing.hive.Ip2Str", "ip2str") + .put("com.linkedin.stdudfs.parsing.hive.UserAgentParser", "useragentparser") + .put("com.linkedin.stdudfs.lookup.hive.BrowserLookup", "browserlookup") + .put("com.linkedin.jobs.udf.hive.ConvertIndustryCode", "converttoindustryv1") + .put("com.linkedin.stdudfs.urnextractor.hive.UrnExtractorFunctionWrapper", "urn_extractor") + .put("com.linkedin.stdudfs.hive.daliudfs.UrnExtractorFunctionWrapper", "urn_extractor") + .put("com.linkedin.groot.runtime.udf.spark.HasMemberConsentUDF", "has_member_consent") + .put("com.linkedin.groot.runtime.udf.spark.RedactFieldIfUDF", "redact_field_if") + .put("com.linkedin.groot.runtime.udf.spark.RedactSecondarySchemaFieldIfUDF", "redact_secondary_schema_field_if") + .put("com.linkedin.groot.runtime.udf.spark.GetMappedValueUDF", "get_mapped_value") + .put("com.linkedin.coral.hive.hive2rel.CoralTestUDF", "coral_test").build(); + // The list of dependencies specified found in the view's "dependencies" property. // Example: "ivy://com.linkedin.udf-group:udf-artifact:0.1.8" private final List ivyDependencies; // The view-dependent function name in the format of "dbName_viewName_functionName", // where functionName is defined in the "functions" property of the view. - private final String viewDependentFunctionName; + private final String originalViewTextFunctionName; + + // The UDF class name value defined in the "functions" property of the view. + // i.e. "functions = : " + private final String functionClassName; private VersionedSqlUserDefinedFunction(SqlIdentifier opName, SqlReturnTypeInference returnTypeInference, SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker, - List paramTypes, Function function, List ivyDependencies, String viewDependentFunctionName) { + List paramTypes, Function function, List ivyDependencies, + String originalViewTextFunctionName, String functionClassName) { super(opName, returnTypeInference, operandTypeInference, operandTypeChecker, paramTypes, function, SqlFunctionCategory.USER_DEFINED_FUNCTION); this.ivyDependencies = ivyDependencies; - this.viewDependentFunctionName = viewDependentFunctionName; - } - - public VersionedSqlUserDefinedFunction(String name, SqlReturnTypeInference returnTypeInference, - SqlOperandTypeChecker operandTypeChecker, List paramTypes, Function function, - List ivyDependencies, String viewDependentFunctionName) { - this(new SqlIdentifier(ImmutableList.of(name), SqlParserPos.ZERO), returnTypeInference, null, operandTypeChecker, - paramTypes, function, ivyDependencies, viewDependentFunctionName); + this.originalViewTextFunctionName = originalViewTextFunctionName; + this.functionClassName = functionClassName; } public VersionedSqlUserDefinedFunction(SqlUserDefinedFunction sqlUdf, List ivyDependencies, - String viewDependentFunctionName) { + String originalViewTextFunctionName, String functionClassName) { this(new SqlIdentifier(ImmutableList.of(sqlUdf.getName()), SqlParserPos.ZERO), sqlUdf.getReturnTypeInference(), null, sqlUdf.getOperandTypeChecker(), sqlUdf.getParamTypes(), sqlUdf.getFunction(), ivyDependencies, - viewDependentFunctionName); + originalViewTextFunctionName, functionClassName); } public List getIvyDependencies() { return ivyDependencies; } - public String getViewDependentFunctionName() { - return viewDependentFunctionName; + public String getOriginalViewTextFunctionName() { + return originalViewTextFunctionName; + } + + /** + * Retrieves the short function name based on the class name. If the class name is found + * in the predefined {@link VersionedSqlUserDefinedFunction#SHORT_FUNC_NAME_MAP}, + * the corresponding short name is returned. Otherwise, the method converts the last + * segment of the class name from UPPER_CAMEL to LOWER_UNDERSCORE format to generate + * the short function name. + */ + public String getShortFunctionName() { + // getName() returns the unversioned function class, which we use to identify the type inference. + // It's just a convention and other naming approaches are valid as long as they identify the type inference. + String unversionedClassName = getName(); + if (SHORT_FUNC_NAME_MAP.containsKey(unversionedClassName)) { + return SHORT_FUNC_NAME_MAP.get(unversionedClassName); + } + Converter caseConverter = CaseFormat.UPPER_CAMEL.converterTo(CaseFormat.LOWER_UNDERSCORE); + String[] nameSplit = unversionedClassName.split("\\."); + return caseConverter.convert(nameSplit[nameSplit.length - 1]); + } + + public String getFunctionClassName() { + return functionClassName; } // This method is called during SQL validation. The super-class implementation resets the call's sqlOperator to one // that is looked up from the StaticHiveFunctionRegistry or inferred dynamically if it's a Dali UDF. Since UDFs in the StaticHiveFunctionRegistry are not // versioned, this method overrides the super-class implementation to properly restore the call's operator as // a VersionedSqlUserDefinedFunction based on the already existing call's sqlOperator obtained from the - // StaticHiveFunctionRegistry, and hence preserve ivyDependencies and viewDependentFunctionName. + // StaticHiveFunctionRegistry, and hence preserve ivyDependencies and originalViewTextFunctionName. @Override public RelDataType deriveType(SqlValidator validator, SqlValidatorScope scope, SqlCall call) { RelDataType relDataType = super.deriveType(validator, scope, call); ((SqlBasicCall) call).setOperator(new VersionedSqlUserDefinedFunction((SqlUserDefinedFunction) (call.getOperator()), - ivyDependencies, viewDependentFunctionName)); + ivyDependencies, originalViewTextFunctionName, functionClassName)); return relDataType; } } diff --git a/coral-hive/src/test/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverterTest.java b/coral-hive/src/test/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverterTest.java index e2eafb0e7..d15240751 100644 --- a/coral-hive/src/test/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverterTest.java +++ b/coral-hive/src/test/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverterTest.java @@ -356,6 +356,14 @@ public void testDaliUDFCall() { assertEquals(RelOptUtil.toString(rel), expectedPlan); } + @Test + public void testVersioningUDF() { + RelNode rel = converter.convertView("test", "tableOneViewShadePrefixUDF"); + String expectedPlan = "LogicalProject(EXPR$0=[com.linkedin.coral.hive.hive2rel.CoralTestUDF($0)])\n" + + " LogicalTableScan(table=[[hive, test, tableone]])\n"; + assertEquals(RelOptUtil.toString(rel), expectedPlan); + } + @Test(expectedExceptions = UnknownSqlFunctionException.class) public void testUnresolvedUdfError() { final String sql = "SELECT default_foo_IsTestMemberId(a) from foo"; diff --git a/coral-hive/src/test/java/com/linkedin/coral/hive/hive2rel/TestUtils.java b/coral-hive/src/test/java/com/linkedin/coral/hive/hive2rel/TestUtils.java index 5a6a9fe0a..32a4572a7 100644 --- a/coral-hive/src/test/java/com/linkedin/coral/hive/hive2rel/TestUtils.java +++ b/coral-hive/src/test/java/com/linkedin/coral/hive/hive2rel/TestUtils.java @@ -194,6 +194,14 @@ public static TestHive setupDefaultHive(HiveConf conf) throws IOException { throw new RuntimeException("Failed to setup view"); } + driver.run( + "create function lessThanHundred_with_versioning_prefix as 'coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel.CoralTestUDF'"); + response = driver.run( + "CREATE VIEW IF NOT EXISTS test.tableOneViewShadePrefixUDF as SELECT lessThanHundred_with_versioning_prefix(a) from test.tableOne"); + if (response.getResponseCode() != 0) { + throw new RuntimeException("Failed to setup view"); + } + driver.run( "CREATE TABLE IF NOT EXISTS union_table(foo uniontype, struct>)"); @@ -234,8 +242,12 @@ public static TestHive setupDefaultHive(HiveConf conf) throws IOException { setOrUpdateDaliFunction(tableOneView, "LessThanHundred", "com.linkedin.coral.hive.hive2rel.CoralTestUDF"); Table tableOneViewLateralUDTF = msc.getTable("test", "tableOneViewLateralUDTF"); setOrUpdateDaliFunction(tableOneViewLateralUDTF, "CountOfRow", "com.linkedin.coral.hive.hive2rel.CoralTestUDTF"); + Table tableOneViewShadePrefixUDF = msc.getTable("test", "tableOneViewShadePrefixUDF"); + setOrUpdateDaliFunction(tableOneViewShadePrefixUDF, "lessThanHundred_with_versioning_prefix", + "coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel.CoralTestUDF"); msc.alter_table("test", "tableOneView", tableOneView); msc.alter_table("test", "tableOneViewLateralUDTF", tableOneViewLateralUDTF); + msc.alter_table("test", "tableOneViewShadePrefixUDF", tableOneViewShadePrefixUDF); hive = testHive; return hive; } catch (Exception e) { diff --git a/coral-hive/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestUDF.java b/coral-hive/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestUDF.java new file mode 100644 index 000000000..0ce9aa265 --- /dev/null +++ b/coral-hive/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestUDF.java @@ -0,0 +1,17 @@ +/** + * Copyright 2018-2024 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel; + +import org.apache.hadoop.hive.ql.exec.UDF; + + +// This is used in TestUtils to set up as dali function +// This needs in a separate file for Hive to correctly load for setup +public class CoralTestUDF extends UDF { + public boolean evaluate(int input) { + return input < 100; + } +} diff --git a/coral-pig/src/main/java/com/linkedin/coral/pig/rel2pig/rel/functions/PigUDF.java b/coral-pig/src/main/java/com/linkedin/coral/pig/rel2pig/rel/functions/PigUDF.java index c667183a7..ea419fd91 100644 --- a/coral-pig/src/main/java/com/linkedin/coral/pig/rel2pig/rel/functions/PigUDF.java +++ b/coral-pig/src/main/java/com/linkedin/coral/pig/rel2pig/rel/functions/PigUDF.java @@ -1,5 +1,5 @@ /** - * Copyright 2019-2021 LinkedIn Corporation. All rights reserved. + * Copyright 2019-2024 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -166,7 +166,7 @@ private String getVersionedFunctionName(RexCall rexCall) { } final VersionedSqlUserDefinedFunction versionedFunction = (VersionedSqlUserDefinedFunction) rexCall.getOperator(); - return String.join("_", PIG_UDF_ALIAS_TEMPLATE, versionedFunction.getViewDependentFunctionName()) + return String.join("_", PIG_UDF_ALIAS_TEMPLATE, versionedFunction.getOriginalViewTextFunctionName()) .replace(NOT_ALPHA_NUMERIC_UNDERSCORE_REGEX, "_"); } diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/HiveUDFTransformer.java b/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/HiveUDFTransformer.java index 38406f942..ef7d6d510 100644 --- a/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/HiveUDFTransformer.java +++ b/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/HiveUDFTransformer.java @@ -63,16 +63,16 @@ protected SqlCall transform(SqlCall sqlCall) { if (UNSUPPORTED_HIVE_UDFS.contains(operatorName)) { throw new UnsupportedUDFException(operatorName); } - final String viewDependentFunctionName = operator.getViewDependentFunctionName(); + final String originalViewTextFunctionName = operator.getOriginalViewTextFunctionName(); final List dependencies = operator.getIvyDependencies(); List listOfUris = dependencies.stream().map(URI::create).collect(Collectors.toList()); LOG.info("Function: {} is not a Builtin UDF or Transport UDF. We fall back to its Hive " + "function with ivy dependency: {}", operatorName, String.join(",", dependencies)); - final SparkUDFInfo sparkUDFInfo = - new SparkUDFInfo(operatorName, viewDependentFunctionName, listOfUris, SparkUDFInfo.UDFTYPE.HIVE_CUSTOM_UDF); + final SparkUDFInfo sparkUDFInfo = new SparkUDFInfo(operator.getFunctionClassName(), originalViewTextFunctionName, + listOfUris, SparkUDFInfo.UDFTYPE.HIVE_CUSTOM_UDF); sparkUDFInfos.add(sparkUDFInfo); final SqlOperator convertedFunction = - createSqlOperator(viewDependentFunctionName, operator.getReturnTypeInference()); + createSqlOperator(originalViewTextFunctionName, operator.getReturnTypeInference()); return convertedFunction.createCall(sqlCall.getParserPosition(), sqlCall.getOperandList()); } } diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/TransportUDFTransformer.java b/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/TransportUDFTransformer.java index 272fd197d..54b2be17d 100644 --- a/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/TransportUDFTransformer.java +++ b/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/TransportUDFTransformer.java @@ -1,5 +1,5 @@ /** - * Copyright 2018-2023 LinkedIn Corporation. All rights reserved. + * Copyright 2018-2024 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -73,13 +73,13 @@ protected boolean condition(SqlCall sqlCall) { @Override protected SqlCall transform(SqlCall sqlCall) { final VersionedSqlUserDefinedFunction operator = (VersionedSqlUserDefinedFunction) sqlCall.getOperator(); - final String viewDependentFunctionName = operator.getViewDependentFunctionName(); - sparkUDFInfos.add(new SparkUDFInfo(sparkUDFClassName, viewDependentFunctionName, + final String originalViewTextFunctionName = operator.getOriginalViewTextFunctionName(); + sparkUDFInfos.add(new SparkUDFInfo(sparkUDFClassName, originalViewTextFunctionName, Collections.singletonList( URI.create(scalaVersion == ScalaVersion.SCALA_2_11 ? artifactoryUrlSpark211 : artifactoryUrlSpark212)), SparkUDFInfo.UDFTYPE.TRANSPORTABLE_UDF)); final SqlOperator convertedFunction = - createSqlOperator(viewDependentFunctionName, operator.getReturnTypeInference()); + createSqlOperator(originalViewTextFunctionName, operator.getReturnTypeInference()); return convertedFunction.createCall(sqlCall.getParserPosition(), sqlCall.getOperandList()); } diff --git a/coral-spark/src/test/java/com/linkedin/coral/spark/CoralSparkTest.java b/coral-spark/src/test/java/com/linkedin/coral/spark/CoralSparkTest.java index 927b499bd..7f2b14e81 100644 --- a/coral-spark/src/test/java/com/linkedin/coral/spark/CoralSparkTest.java +++ b/coral-spark/src/test/java/com/linkedin/coral/spark/CoralSparkTest.java @@ -48,6 +48,8 @@ public void beforeClass() throws HiveException, MetaException, IOException { // add the following 3 test UDF to StaticHiveFunctionRegistry for testing purpose. StaticHiveFunctionRegistry.createAddUserDefinedFunction("com.linkedin.coral.hive.hive2rel.CoralTestUDF", ReturnTypes.BOOLEAN, family(SqlTypeFamily.INTEGER)); + StaticHiveFunctionRegistry.createAddUserDefinedFunction("com.linkedin.coral.hive.hive2rel.CoralTestVersionedUDF", + ReturnTypes.BOOLEAN, family(SqlTypeFamily.INTEGER)); StaticHiveFunctionRegistry.createAddUserDefinedFunction("com.linkedin.coral.hive.hive2rel.CoralTestUDF2", ReturnTypes.BOOLEAN, family(SqlTypeFamily.INTEGER)); StaticHiveFunctionRegistry.createAddUserDefinedFunction("com.linkedin.coral.hive.hive2rel.CoralTestUdfSquare", @@ -164,6 +166,38 @@ public void testHiveUDFTransformer() { assertEquals(sparkSqlStmt, targetSqlStmt); } + @Test + public void testVersioningUDF() { + // After registering unversioned UDF "com.linkedin.coral.hive.hive2rel.CoralTestUDF", + // Coral should be able to translate the view with the corresponding versioned UDF class + // "coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel.CoralTestUDF" + RelNode relNode = TestUtils.toRelNode("default", "foo_udf_with_versioning_prefix"); + CoralSpark coralSpark = createCoralSpark(relNode); + List udfJars = coralSpark.getSparkUDFInfoList(); + + // Shaded UDF class name should be returned for UDF registration, otherwise + // Spark can't find the unversioned UDF name in the shaded Jar. + String udfClassName = udfJars.get(0).getClassName(); + String targetClassName = "coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel.CoralTestVersionedUDF"; + assertEquals(udfClassName, targetClassName); + + String udfFunctionName = udfJars.get(0).getFunctionName(); + String targetFunctionName = "LessThanHundred_versioning_prefix_0_1_x"; + assertEquals(udfFunctionName, targetFunctionName); + + List listOfUriStrings = convertToListOfUriStrings(udfJars.get(0).getArtifactoryUrls()); + String targetArtifactoryUrl = "ivy://com.linkedin:udf-shaded:1.0"; + assertTrue(listOfUriStrings.contains(targetArtifactoryUrl)); + + SparkUDFInfo.UDFTYPE testUdfType = udfJars.get(0).getUdfType(); + SparkUDFInfo.UDFTYPE targetUdfType = SparkUDFInfo.UDFTYPE.HIVE_CUSTOM_UDF; + assertEquals(testUdfType, targetUdfType); + + String sparkSqlStmt = coralSpark.getSparkSql(); + String targetSqlStmt = "SELECT LessThanHundred_versioning_prefix_0_1_x(foo.a)\n" + "FROM default.foo foo"; + assertEquals(sparkSqlStmt, targetSqlStmt); + } + @Test(expectedExceptions = UnsupportedUDFException.class) public void testUnsupportedUdf() { RelNode relNode = TestUtils.toRelNode("default", "foo_dali_udf5"); diff --git a/coral-spark/src/test/java/com/linkedin/coral/spark/TestUtils.java b/coral-spark/src/test/java/com/linkedin/coral/spark/TestUtils.java index 7f4ae0617..23ec04c8f 100644 --- a/coral-spark/src/test/java/com/linkedin/coral/spark/TestUtils.java +++ b/coral-spark/src/test/java/com/linkedin/coral/spark/TestUtils.java @@ -83,6 +83,9 @@ public static void initializeViews(HiveConf conf) throws HiveException, MetaExce "create function default_foo_duplicate_udf_LessThanHundred as 'com.linkedin.coral.hive.hive2rel.CoralTestUDF'"); run(driver, "CREATE FUNCTION LessThanHundred as 'com.linkedin.coral.hive.hive2rel.CoralTestUDF'"); + run(driver, + "CREATE FUNCTION LessThanHundred_versioning_prefix_0_1_x as 'coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel.CoralTestVersionedUDF'"); + run(driver, String.join("\n", "", "CREATE VIEW IF NOT EXISTS foo_view", "AS", "SELECT b AS bcol, sum(c) AS sum_c", "FROM foo", "GROUP BY b")); run(driver, "DROP VIEW IF EXISTS foo_v1"); @@ -235,6 +238,11 @@ public static void initializeViews(HiveConf conf) throws HiveException, MetaExce " 'dependencies' = 'ivy://com.linkedin:udf:1.0')", "AS", "SELECT default_foo_duplicate_udf_LessThanHundred(a), default_foo_duplicate_udf_LessThanHundred(a)", "FROM foo")); + + run(driver, String.join("\n", "", "CREATE VIEW IF NOT EXISTS foo_udf_with_versioning_prefix", + "tblproperties('functions' = 'LessThanHundred_versioning_prefix_0_1_x:coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel.CoralTestVersionedUDF',", + " 'dependencies' = 'ivy://com.linkedin:udf-shaded:1.0')", "AS", + "SELECT LessThanHundred_versioning_prefix_0_1_x(a)", "FROM foo")); } private static void executeCreateTableQuery(Driver driver, String dbName, String tableName, String schema) { diff --git a/coral-spark/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestVersionedUDF.java b/coral-spark/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestVersionedUDF.java new file mode 100644 index 000000000..8c236d78d --- /dev/null +++ b/coral-spark/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestVersionedUDF.java @@ -0,0 +1,17 @@ +/** + * Copyright 2018-2024 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel; + +import org.apache.hadoop.hive.ql.exec.UDF; + + +// This is used in TestUtils to set up as dali function +// This needs in a separate file for Hive to correctly load for setup +public class CoralTestVersionedUDF extends UDF { + public boolean evaluate(int input) { + return input < 100; + } +} diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/CoralToTrinoSqlCallConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/CoralToTrinoSqlCallConverter.java index 510282e9d..ca2cde7f6 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/CoralToTrinoSqlCallConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/CoralToTrinoSqlCallConverter.java @@ -28,7 +28,7 @@ import com.linkedin.coral.trino.rel2trino.transformers.CoralRegistryOperatorRenameSqlCallTransformer; import com.linkedin.coral.trino.rel2trino.transformers.CurrentTimestampTransformer; import com.linkedin.coral.trino.rel2trino.transformers.FromUnixtimeOperatorTransformer; -import com.linkedin.coral.trino.rel2trino.transformers.GenericCoralRegistryOperatorRenameSqlCallTransformer; +import com.linkedin.coral.trino.rel2trino.transformers.HiveUDFTransformer; import com.linkedin.coral.trino.rel2trino.transformers.JoinSqlCallTransformer; import com.linkedin.coral.trino.rel2trino.transformers.MapValueConstructorTransformer; import com.linkedin.coral.trino.rel2trino.transformers.NullOrderingTransformer; @@ -107,31 +107,7 @@ protected SqlCall transform(SqlCall sqlCall) { new ToDateOperatorTransformer(configs.getOrDefault(AVOID_TRANSFORM_TO_DATE_UDF, false)), new CurrentTimestampTransformer(), new FromUnixtimeOperatorTransformer(), - // LinkedIn specific functions - new CoralRegistryOperatorRenameSqlCallTransformer( - "com.linkedin.dali.udf.watbotcrawlerlookup.hive.WATBotCrawlerLookup", 3, "wat_bot_crawler_lookup"), - new CoralRegistryOperatorRenameSqlCallTransformer("com.linkedin.stdudfs.parsing.hive.Ip2Str", 1, "ip2str"), - new CoralRegistryOperatorRenameSqlCallTransformer("com.linkedin.stdudfs.parsing.hive.Ip2Str", 3, "ip2str"), - new CoralRegistryOperatorRenameSqlCallTransformer("com.linkedin.stdudfs.parsing.hive.UserAgentParser", 2, - "useragentparser"), - new CoralRegistryOperatorRenameSqlCallTransformer("com.linkedin.stdudfs.lookup.hive.BrowserLookup", 3, - "browserlookup"), - new CoralRegistryOperatorRenameSqlCallTransformer("com.linkedin.jobs.udf.hive.ConvertIndustryCode", 1, - "converttoindustryv1"), - new CoralRegistryOperatorRenameSqlCallTransformer( - "com.linkedin.stdudfs.urnextractor.hive.UrnExtractorFunctionWrapper", 1, "urn_extractor"), - new CoralRegistryOperatorRenameSqlCallTransformer( - "com.linkedin.stdudfs.hive.daliudfs.UrnExtractorFunctionWrapper", 1, "urn_extractor"), - new CoralRegistryOperatorRenameSqlCallTransformer("com.linkedin.groot.runtime.udf.spark.HasMemberConsentUDF", 3, - "has_member_consent"), - new CoralRegistryOperatorRenameSqlCallTransformer("com.linkedin.groot.runtime.udf.spark.RedactFieldIfUDF", 4, - "redact_field_if"), - new CoralRegistryOperatorRenameSqlCallTransformer( - "com.linkedin.groot.runtime.udf.spark.RedactSecondarySchemaFieldIfUDF", 5, - "redact_secondary_schema_field_if"), - new CoralRegistryOperatorRenameSqlCallTransformer("com.linkedin.groot.runtime.udf.spark.GetMappedValueUDF", 2, - "get_mapped_value"), - new GenericCoralRegistryOperatorRenameSqlCallTransformer(), + new HiveUDFTransformer(), new ReturnTypeAdjustmentTransformer(configs), new UnnestOperatorTransformer(), new AsOperatorTransformer(), new JoinSqlCallTransformer(), new NullOrderingTransformer(), new SubstrIndexTransformer()); diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/CoralRegistryOperatorRenameSqlCallTransformer.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/CoralRegistryOperatorRenameSqlCallTransformer.java index b9a661450..562e823ef 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/CoralRegistryOperatorRenameSqlCallTransformer.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/CoralRegistryOperatorRenameSqlCallTransformer.java @@ -1,5 +1,5 @@ /** - * Copyright 2023 LinkedIn Corporation. All rights reserved. + * Copyright 2023-2024 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/GenericCoralRegistryOperatorRenameSqlCallTransformer.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/GenericCoralRegistryOperatorRenameSqlCallTransformer.java deleted file mode 100644 index 34e23f8c7..000000000 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/GenericCoralRegistryOperatorRenameSqlCallTransformer.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2023 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.coral.trino.rel2trino.transformers; - -import org.apache.calcite.sql.SqlCall; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.parser.SqlParserPos; - -import com.linkedin.coral.com.google.common.base.CaseFormat; -import com.linkedin.coral.com.google.common.base.Converter; -import com.linkedin.coral.common.transformers.SqlCallTransformer; -import com.linkedin.coral.hive.hive2rel.functions.StaticHiveFunctionRegistry; - - -/** - * This is a subclass of {@link SqlCallTransformer}. It transforms any LinkedIn specific Coral operator - * to Trino operator which are not handled by {@link CoralRegistryOperatorRenameSqlCallTransformer}. - * e.g. from "com.linkedin.dali.udf.IsTestMemberId"("id") to "is_test_member_id("id") - */ -public class GenericCoralRegistryOperatorRenameSqlCallTransformer extends SqlCallTransformer { - - private static final StaticHiveFunctionRegistry HIVE_FUNCTION_REGISTRY = new StaticHiveFunctionRegistry(); - - @Override - protected boolean condition(SqlCall sqlCall) { - return sqlCall.getOperator().getName().startsWith("com.linkedin"); - } - - @Override - protected SqlCall transform(SqlCall sqlCall) { - Converter caseConverter = CaseFormat.UPPER_CAMEL.converterTo(CaseFormat.LOWER_UNDERSCORE); - SqlOperator sourceOp = HIVE_FUNCTION_REGISTRY.getRegistry().containsKey(sqlCall.getOperator().getName()) - ? HIVE_FUNCTION_REGISTRY.lookup(sqlCall.getOperator().getName()).iterator().next().getSqlOperator() - : sqlCall.getOperator(); - String[] nameSplit = sourceOp.getName().split("\\."); - String targetName = caseConverter.convert(nameSplit[nameSplit.length - 1]); - SqlOperator targetOp = createSqlOperator(targetName, sourceOp.getReturnTypeInference()); - return targetOp.createCall(new SqlNodeList(sqlCall.getOperandList(), SqlParserPos.ZERO)); - } -} diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/HiveUDFTransformer.java b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/HiveUDFTransformer.java new file mode 100644 index 000000000..0d48fd540 --- /dev/null +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/rel2trino/transformers/HiveUDFTransformer.java @@ -0,0 +1,39 @@ +/** + * Copyright 2023-2024 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.coral.trino.rel2trino.transformers; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.parser.SqlParserPos; + +import com.linkedin.coral.common.transformers.SqlCallTransformer; +import com.linkedin.coral.hive.hive2rel.functions.VersionedSqlUserDefinedFunction; + + +/** + * This transformer converts the Hive UDF SqlCall name from the UDF class name to the + * corresponding Trino function name. + * i.e. from `com.linkedin.stdudfs.parsing.hive.Ip2Str` to `ip2str`. + */ +public class HiveUDFTransformer extends SqlCallTransformer { + + @Override + protected boolean condition(SqlCall sqlCall) { + final SqlOperator operator = sqlCall.getOperator(); + final String operatorName = operator.getName(); + return operator instanceof VersionedSqlUserDefinedFunction && operatorName.contains(".") + && !operatorName.equals("."); + } + + @Override + protected SqlCall transform(SqlCall sqlCall) { + final SqlOperator operator = sqlCall.getOperator(); + final String trinoFunctionName = ((VersionedSqlUserDefinedFunction) operator).getShortFunctionName(); + return createSqlOperator(trinoFunctionName, operator.getReturnTypeInference()) + .createCall(new SqlNodeList(sqlCall.getOperandList(), SqlParserPos.ZERO)); + } +} diff --git a/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/HiveToTrinoConverterTest.java b/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/HiveToTrinoConverterTest.java index 7f5cb2ce9..6bec9b880 100644 --- a/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/HiveToTrinoConverterTest.java +++ b/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/HiveToTrinoConverterTest.java @@ -11,6 +11,8 @@ import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -20,7 +22,10 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.linkedin.coral.hive.hive2rel.functions.StaticHiveFunctionRegistry; + import static com.linkedin.coral.trino.rel2trino.CoralTrinoConfigKeys.*; +import static org.apache.calcite.sql.type.OperandTypes.*; import static org.testng.Assert.assertEquals; @@ -32,6 +37,8 @@ public class HiveToTrinoConverterTest { public void beforeClass() throws IOException, HiveException, MetaException { conf = TestUtils.loadResourceHiveConf(); TestUtils.initializeTablesAndViews(conf); + StaticHiveFunctionRegistry.createAddUserDefinedFunction("com.linkedin.coral.hive.hive2rel.CoralTestUDF", + ReturnTypes.BOOLEAN, family(SqlTypeFamily.INTEGER)); } @AfterTest @@ -994,4 +1001,14 @@ public void testSqlSelectAliasAppenderTransformerWithoutTableAliasPrefix() { + "WHERE \"tablea\".\"a\" > 5"; assertEquals(expandedSql, expected); } + + @Test + public void testVersioningUDF() { + RelNode relNode = TestUtils.getHiveToRelConverter().convertView("test", "udf_with_versioning_prefix"); + RelToTrinoConverter relToTrinoConverter = TestUtils.getRelToTrinoConverter(); + String expandedSql = relToTrinoConverter.convert(relNode); + + String expected = "SELECT \"coral_test\"(\"tablea\".\"a\")\n" + "FROM \"test\".\"tablea\" AS \"tablea\""; + assertEquals(expandedSql, expected); + } } diff --git a/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/TestUtils.java b/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/TestUtils.java index 3e6f088f8..2bb368e13 100644 --- a/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/TestUtils.java +++ b/coral-trino/src/test/java/com/linkedin/coral/trino/rel2trino/TestUtils.java @@ -413,6 +413,14 @@ public static void initializeTablesAndViews(HiveConf conf) throws HiveException, + "UNION ALL\n" + "SELECT a_tinyint, a_smallint, a_integer, a_bigint, a_float FROM test.table_with_mixed_columns"); + run(driver, + "CREATE FUNCTION LessThanHundred_versioning_prefix AS 'coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel.CoralTestUDF'"); + + run(driver, String.join("\n", "CREATE VIEW IF NOT EXISTS test.udf_with_versioning_prefix", + "tblproperties('functions' = 'LessThanHundred_versioning_prefix:coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel.CoralTestUDF',", + " 'dependencies' = 'ivy://com.linkedin:udf-shaded:1.0')", "AS", + "SELECT LessThanHundred_versioning_prefix(a)", "FROM test.tableA")); + // Tables used in RelToTrinoConverterTest run(driver, "CREATE TABLE IF NOT EXISTS test.tableOne(icol int, dcol double, scol string, tcol timestamp, acol array)"); diff --git a/coral-trino/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestUDF.java b/coral-trino/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestUDF.java new file mode 100644 index 000000000..0ce9aa265 --- /dev/null +++ b/coral-trino/src/test/java/coral_udf_version_0_1_x/com/linkedin/coral/hive/hive2rel/CoralTestUDF.java @@ -0,0 +1,17 @@ +/** + * Copyright 2018-2024 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package coral_udf_version_0_1_x.com.linkedin.coral.hive.hive2rel; + +import org.apache.hadoop.hive.ql.exec.UDF; + + +// This is used in TestUtils to set up as dali function +// This needs in a separate file for Hive to correctly load for setup +public class CoralTestUDF extends UDF { + public boolean evaluate(int input) { + return input < 100; + } +}