diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/digest/DigestGenerationHandler.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/digest/DigestGenerationHandler.java index 8cd64ed22a6..17bec00df1d 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/digest/DigestGenerationHandler.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/digest/DigestGenerationHandler.java @@ -14,8 +14,8 @@ package org.finos.legend.engine.persistence.components.ingestmode.digest; -import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.logicalplan.values.DigestUdf; import org.finos.legend.engine.persistence.components.logicalplan.values.FieldValue; import org.finos.legend.engine.persistence.components.logicalplan.values.StagedFilesFieldValue; @@ -30,10 +30,10 @@ public class DigestGenerationHandler implements DigestGenStrategyVisitor { private List fieldsToSelect; private List fieldsToInsert; - private List fieldTypes; + private List fieldTypes; private Dataset mainDataset; - public DigestGenerationHandler(Dataset mainDataset, List fieldsToSelect, List fieldsToInsert, List fieldTypes) + public DigestGenerationHandler(Dataset mainDataset, List fieldsToSelect, List fieldsToInsert, List fieldTypes) { this.mainDataset = mainDataset; this.fieldsToSelect = fieldsToSelect; @@ -53,7 +53,7 @@ public Void visitUDFBasedDigestGenStrategy(UDFBasedDigestGenStrategyAbstract udf Set fieldsToExclude = udfBasedDigestGenStrategy.fieldsToExcludeFromDigest(); List filteredStagingFieldNames = new ArrayList<>(); List filteredStagingFieldValues = new ArrayList<>(); - List filteredStagingFieldTypes = new ArrayList<>(); + List filteredStagingFieldTypes = new ArrayList<>(); List sortedFieldsToSelect = fieldsToSelect.stream().sorted((o1, o2) -> { @@ -71,7 +71,7 @@ else if (o1 instanceof StagedFilesFieldValue && o2 instanceof StagedFilesFieldVa for (Value value : sortedFieldsToSelect) { int index = fieldsToSelect.indexOf(value); - DataType dataType = fieldTypes.get(index); + FieldType dataType = fieldTypes.get(index); if (value instanceof FieldValue) { diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/CastFunctionAbstract.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/CastFunctionAbstract.java new file mode 100644 index 00000000000..9392c993c96 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/CastFunctionAbstract.java @@ -0,0 +1,38 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.logicalplan.values; + +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; + +import static org.immutables.value.Value.Immutable; +import static org.immutables.value.Value.Parameter; +import static org.immutables.value.Value.Style; + +@Immutable +@Style( + typeAbstract = "*Abstract", + typeImmutable = "*", + jdkOnly = true, + optionalAcceptNullable = true, + strictBuilder = true +) +public interface CastFunctionAbstract extends Value +{ + @Parameter(order = 0) + Value field(); + + @Parameter(order = 1) + FieldType type(); +} diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/DigestUdfAbstract.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/DigestUdfAbstract.java index 7a7efae90ad..611eed08165 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/DigestUdfAbstract.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/DigestUdfAbstract.java @@ -15,6 +15,7 @@ package org.finos.legend.engine.persistence.components.logicalplan.values; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import java.util.List; import java.util.Map; @@ -36,7 +37,7 @@ public interface DigestUdfAbstract extends Value List values(); - List fieldTypes(); + List fieldTypes(); Map typeConversionUdfNames(); } diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/AppendOnlyPlanner.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/AppendOnlyPlanner.java index 26cefc6f3a1..92448d6f052 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/AppendOnlyPlanner.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/AppendOnlyPlanner.java @@ -31,9 +31,9 @@ import org.finos.legend.engine.persistence.components.logicalplan.conditions.Condition; import org.finos.legend.engine.persistence.components.logicalplan.conditions.Exists; import org.finos.legend.engine.persistence.components.logicalplan.conditions.Not; -import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Selection; import org.finos.legend.engine.persistence.components.logicalplan.operations.Insert; import org.finos.legend.engine.persistence.components.logicalplan.values.All; @@ -107,7 +107,7 @@ protected AppendOnly ingestMode() @Override public LogicalPlan buildLogicalPlanForIngest(Resources resources) { - Pair, List> dataFieldsWithTypes = getDataFieldsWithTypes(); + Pair, List> dataFieldsWithTypes = getDataFieldsWithTypes(); List dataFields = dataFieldsWithTypes.getOne(); List fieldsToSelect = new ArrayList<>(dataFields); List fieldsToInsert = new ArrayList<>(dataFields); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/BulkLoadPlanner.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/BulkLoadPlanner.java index 3055a562581..d4b04ad14e5 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/BulkLoadPlanner.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/BulkLoadPlanner.java @@ -275,7 +275,7 @@ private LogicalPlan buildLogicalPlanForTransformWhileCopy(Resources resources) List fieldsToInsert = new ArrayList<>(stagingDataset().schemaReference().fieldValues()); // Add digest - List fieldTypes = stagingDataset().schema().fields().stream().map(field -> field.type().dataType()).collect(Collectors.toList()); + List fieldTypes = stagingDataset().schema().fields().stream().map(field -> field.type()).collect(Collectors.toList()); ingestMode().digestGenStrategy().accept(new DigestGenerationHandler(mainDataset(), fieldsToSelect, fieldsToInsert, fieldTypes)); // Add batch_id field @@ -304,7 +304,7 @@ private LogicalPlan buildLogicalPlanForCopyAndTransform(Resources resources) List fieldsToInsertIntoMain = new ArrayList<>(externalDataset.schemaReference().fieldValues()); // Add digest - List fieldTypes = externalDataset.schema().fields().stream().map(field -> field.type().dataType()).collect(Collectors.toList()); + List fieldTypes = externalDataset.schema().fields().stream().map(field -> field.type()).collect(Collectors.toList()); ingestMode().digestGenStrategy().accept(new DigestGenerationHandler(mainDataset(), fieldsToSelect, fieldsToInsertIntoMain, fieldTypes)); // Add batch_id field diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/Planner.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/Planner.java index d444b1dbfc0..01424d91504 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/Planner.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/Planner.java @@ -35,10 +35,10 @@ import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanFactory; import org.finos.legend.engine.persistence.components.logicalplan.conditions.Condition; import org.finos.legend.engine.persistence.components.logicalplan.conditions.GreaterThan; -import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DerivedDataset; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.logicalplan.datasets.FilteredDataset; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Selection; import org.finos.legend.engine.persistence.components.logicalplan.operations.*; @@ -239,10 +239,10 @@ protected Dataset tempStagingDatasetWithoutPks() return tempStagingDatasetWithoutPks.orElseThrow(IllegalStateException::new); } - protected Pair, List> getDataFieldsWithTypes() + protected Pair, List> getDataFieldsWithTypes() { List dataFields = new ArrayList<>(); - List fieldTypes = new ArrayList<>(); + List fieldTypes = new ArrayList<>(); Optional dedupField = ingestMode.deduplicationStrategy().accept(DeduplicationVisitors.EXTRACT_DEDUP_FIELD); Optional dataSplitField = ingestMode.dataSplitField(); @@ -256,7 +256,7 @@ protected Pair, List> getDataFieldsWithTypes() continue; } dataFields.add(fieldValue); - fieldTypes.add(stagingDataset().schema().fields().get(i).type().dataType()); + fieldTypes.add(stagingDataset().schema().fields().get(i).type()); } return Tuples.pair(dataFields, fieldTypes); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/DigestUdfVisitor.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/DigestUdfVisitor.java index 55dab1bef5f..9e65b0fd529 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/DigestUdfVisitor.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/DigestUdfVisitor.java @@ -15,6 +15,7 @@ package org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.logicalplan.values.DigestUdf; import org.finos.legend.engine.persistence.components.logicalplan.values.StringValue; import org.finos.legend.engine.persistence.components.logicalplan.values.ToArrayFunction; @@ -51,7 +52,7 @@ public VisitorResult visit(PhysicalPlanNode prev, DigestUdf current, VisitorCont return new VisitorResult(udf, Arrays.asList(toArrayColumnNames, toArrayColumnValues)); } - protected Value getColumnValueAsStringType(Value value, DataType dataType, Map typeConversionUdfNames) + protected Value getColumnValueAsStringType(Value value, FieldType dataType, Map typeConversionUdfNames) { throw new IllegalStateException("UDF is unsupported in ANSI sink"); } diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/DigestUdfVisitor.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/DigestUdfVisitor.java index fc9c3f42ccf..05f474e6e13 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/DigestUdfVisitor.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/DigestUdfVisitor.java @@ -15,6 +15,7 @@ package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.logicalplan.values.Value; import java.util.Map; @@ -23,11 +24,11 @@ public class DigestUdfVisitor extends org.finos.legend.engine.persistence.compon { @Override - protected Value getColumnValueAsStringType(Value value, DataType dataType, Map typeConversionUdfNames) + protected Value getColumnValueAsStringType(Value value, FieldType dataType, Map typeConversionUdfNames) { - if (typeConversionUdfNames.containsKey(dataType)) + if (typeConversionUdfNames.containsKey(dataType.dataType())) { - return org.finos.legend.engine.persistence.components.logicalplan.values.Udf.builder().udfName(typeConversionUdfNames.get(dataType)).addParameters(value).build(); + return org.finos.legend.engine.persistence.components.logicalplan.values.Udf.builder().udfName(typeConversionUdfNames.get(dataType.dataType())).addParameters(value).build(); } else { diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/visitor/DigestUdfVisitor.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/visitor/DigestUdfVisitor.java index 51018d00de9..800ac27937a 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/visitor/DigestUdfVisitor.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/visitor/DigestUdfVisitor.java @@ -15,6 +15,7 @@ package org.finos.legend.engine.persistence.components.relational.h2.sql.visitor; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.logicalplan.values.FieldValue; import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionImpl; import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionName; @@ -27,7 +28,7 @@ public class DigestUdfVisitor extends org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.DigestUdfVisitor { - protected Value getColumnValueAsStringType(Value value, DataType dataType, Map typeConversionUdfNames) + protected Value getColumnValueAsStringType(Value value, FieldType dataType, Map typeConversionUdfNames) { if (value instanceof StagedFilesFieldValue) { @@ -37,9 +38,9 @@ protected Value getColumnValueAsStringType(Value value, DataType dataType, Map +{ + @Override + public VisitorResult visit(PhysicalPlanNode prev, CastFunction current, VisitorContext context) + { + DataType dataType = new SnowflakeDataTypeMapping().getDataType(current.type()); + + org.finos.legend.engine.persistence.components.relational.snowflake.sqldom.schemaops.values.CastFunction castFunction = + new org.finos.legend.engine.persistence.components.relational.snowflake.sqldom.schemaops.values.CastFunction(dataType, context.quoteIdentifier()); + for (Optimizer optimizer : context.optimizers()) + { + castFunction = (org.finos.legend.engine.persistence.components.relational.snowflake.sqldom.schemaops.values.CastFunction) optimizer.optimize(castFunction); + } + prev.push(castFunction); + + List logicalPlanNodeList = new ArrayList<>(); + logicalPlanNodeList.add(current.field()); + + return new VisitorResult(castFunction, logicalPlanNodeList); + } +} diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/DigestUdfVisitor.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/DigestUdfVisitor.java index 77c3985ebbe..87718675bff 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/DigestUdfVisitor.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/DigestUdfVisitor.java @@ -15,21 +15,42 @@ package org.finos.legend.engine.persistence.components.relational.snowflake.sql.visitor; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.values.CastFunction; +import org.finos.legend.engine.persistence.components.logicalplan.values.StagedFilesFieldValue; import org.finos.legend.engine.persistence.components.logicalplan.values.Value; import java.util.Map; public class DigestUdfVisitor extends org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.DigestUdfVisitor { - protected Value getColumnValueAsStringType(Value value, DataType dataType, Map typeConversionUdfNames) + protected Value getColumnValueAsStringType(Value value, FieldType dataType, Map typeConversionUdfNames) { - if (typeConversionUdfNames.containsKey(dataType)) + if (value instanceof StagedFilesFieldValue) { - return org.finos.legend.engine.persistence.components.logicalplan.values.Udf.builder().udfName(typeConversionUdfNames.get(dataType)).addParameters(value).build(); + if (typeConversionUdfNames.containsKey(dataType.dataType())) + { + // TO_STRING(CAST(field)) + return org.finos.legend.engine.persistence.components.logicalplan.values.Udf.builder().udfName(typeConversionUdfNames.get(dataType.dataType())).addParameters(CastFunction.builder().field(value).type(dataType).build()).build(); + } + else + { + // CAST(field) + return CastFunction.builder().field(value).type(dataType).build(); + } } else { - return value; + if (typeConversionUdfNames.containsKey(dataType.dataType())) + { + // TO_STRING(field) + return org.finos.legend.engine.persistence.components.logicalplan.values.Udf.builder().udfName(typeConversionUdfNames.get(dataType.dataType())).addParameters(value).build(); + } + else + { + // field + return value; + } } } } diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sqldom/schemaops/values/CastFunction.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sqldom/schemaops/values/CastFunction.java new file mode 100644 index 00000000000..68c06fc2534 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sqldom/schemaops/values/CastFunction.java @@ -0,0 +1,69 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.snowflake.sqldom.schemaops.values; + +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlDomException; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.DataType; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.values.Value; + +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.CLOSING_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.OPEN_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.WHITE_SPACE; + +public class CastFunction extends Value +{ + private Value column; + private DataType dataType; + + public CastFunction(DataType dataType, String quoteIdentifier) + { + super(quoteIdentifier); + this.dataType = dataType; + } + + @Override + public void genSql(StringBuilder builder) throws SqlDomException + { + genSqlWithoutAlias(builder); + super.genSql(builder); + } + + @Override + public void genSqlWithoutAlias(StringBuilder builder) throws SqlDomException + { + builder.append(Clause.CAST); + builder.append(OPEN_PARENTHESIS); + column.genSqlWithoutAlias(builder); + builder.append(WHITE_SPACE); + builder.append(Clause.AS); + builder.append(WHITE_SPACE); + dataType.genSql(builder); + builder.append(CLOSING_PARENTHESIS); + } + + @Override + public void push(Object node) + { + if (node instanceof Value) + { + column = (Value) node; + } + else if (node instanceof DataType) + { + dataType = (DataType) node; + } + } +} diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BulkLoadTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BulkLoadTest.java index acf896ecb70..448571aebf0 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BulkLoadTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BulkLoadTest.java @@ -79,6 +79,12 @@ public class BulkLoadTest .columnNumber(5) .build(); + private static Field col5 = Field.builder() + .name("col_decimal") + .type(FieldType.of(DataType.DECIMAL, 4, 1)) + .columnNumber(6) + .build(); + private static Field col1NonNullable = Field.builder() .name("col_int") .type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())) @@ -334,7 +340,7 @@ public void testBulkLoadWithUpperCaseConversionAndNoEventId() "(\"COL_INT\", \"COL_INTEGER\", \"DIGEST\", \"BATCH_ID\", \"APPEND_TIME\") " + "FROM " + "(SELECT legend_persistence_stage.$1 as \"COL_INT\",legend_persistence_stage.$2 as \"COL_INTEGER\"," + - "LAKEHOUSE_MD5(ARRAY_CONSTRUCT('COL_INT','COL_INTEGER'),ARRAY_CONSTRUCT(legend_persistence_stage.$1,legend_persistence_stage.$2))," + + "LAKEHOUSE_MD5(ARRAY_CONSTRUCT('COL_INT','COL_INTEGER'),ARRAY_CONSTRUCT(CAST(legend_persistence_stage.$1 AS INTEGER),CAST(legend_persistence_stage.$2 AS INTEGER)))," + "(SELECT COALESCE(MAX(BATCH_METADATA.\"TABLE_BATCH_ID\"),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.\"TABLE_NAME\") = 'MY_NAME'),'2000-01-01 00:00:00.000000' " + "FROM my_location as legend_persistence_stage) " + "FILES = ('/path/xyz/file1.csv', '/path/xyz/file2.csv') " + @@ -511,7 +517,7 @@ public void testBulkLoadWithDigest() "(\"col_int\", \"col_integer\", \"digest\", \"batch_id\", \"append_time\") " + "FROM " + "(SELECT legend_persistence_stage.$1 as \"col_int\",legend_persistence_stage.$2 as \"col_integer\"," + - "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_int','col_integer'),ARRAY_CONSTRUCT(legend_persistence_stage.$1,legend_persistence_stage.$2))," + + "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_int','col_integer'),ARRAY_CONSTRUCT(CAST(legend_persistence_stage.$1 AS INTEGER),CAST(legend_persistence_stage.$2 AS INTEGER)))," + "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MY_NAME'),'2000-01-01 00:00:00.000000' " + "FROM my_location as legend_persistence_stage) " + "FILES = ('/path/xyz/file1.csv', '/path/xyz/file2.csv') " + @@ -549,7 +555,7 @@ public void testBulkLoadWithDigestAndTypeConversionUdfs() .location("my_location") .fileFormat(UserDefinedFileFormat.of("my_file_format")) .addAllFilePaths(filesList).build()) - .schema(SchemaDefinition.builder().addAllFields(Arrays.asList(col1, col2, col3, col4)).build()) + .schema(SchemaDefinition.builder().addAllFields(Arrays.asList(col1, col2, col3, col4, col5)).build()) .build(); Dataset mainDataset = DatasetDefinition.builder() @@ -572,12 +578,12 @@ public void testBulkLoadWithDigestAndTypeConversionUdfs() List ingestSql = operations.ingestSql(); Map statsSql = operations.postIngestStatisticsSql(); - String expectedCreateTableSql = "CREATE TABLE IF NOT EXISTS \"my_db\".\"my_name\"(\"col_int\" INTEGER,\"col_integer\" INTEGER,\"col_bigint\" BIGINT,\"col_variant\" VARIANT,\"digest\" VARCHAR,\"batch_id\" INTEGER,\"append_time\" DATETIME)"; + String expectedCreateTableSql = "CREATE TABLE IF NOT EXISTS \"my_db\".\"my_name\"(\"col_int\" INTEGER,\"col_integer\" INTEGER,\"col_bigint\" BIGINT,\"col_variant\" VARIANT,\"col_decimal\" NUMBER(4,1),\"digest\" VARCHAR,\"batch_id\" INTEGER,\"append_time\" DATETIME)"; String expectedIngestSql = "COPY INTO \"my_db\".\"my_name\" " + - "(\"col_int\", \"col_integer\", \"col_bigint\", \"col_variant\", \"digest\", \"batch_id\", \"append_time\") FROM " + - "(SELECT legend_persistence_stage.$1 as \"col_int\",legend_persistence_stage.$2 as \"col_integer\",legend_persistence_stage.$3 as \"col_bigint\",TO_VARIANT(PARSE_JSON(legend_persistence_stage.$4)) as \"col_variant\"," + - "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_bigint','col_int','col_integer'),ARRAY_CONSTRUCT(longToString(legend_persistence_stage.$3),intToString(legend_persistence_stage.$1),intToString(legend_persistence_stage.$2)))," + + "(\"col_int\", \"col_integer\", \"col_bigint\", \"col_variant\", \"col_decimal\", \"digest\", \"batch_id\", \"append_time\") FROM " + + "(SELECT legend_persistence_stage.$1 as \"col_int\",legend_persistence_stage.$2 as \"col_integer\",legend_persistence_stage.$3 as \"col_bigint\",TO_VARIANT(PARSE_JSON(legend_persistence_stage.$4)) as \"col_variant\",legend_persistence_stage.$5 as \"col_decimal\"," + + "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_bigint','col_decimal','col_int','col_integer'),ARRAY_CONSTRUCT(longToString(CAST(legend_persistence_stage.$3 AS BIGINT)),CAST(legend_persistence_stage.$5 AS NUMBER(4,1)),intToString(CAST(legend_persistence_stage.$1 AS INTEGER)),intToString(CAST(legend_persistence_stage.$2 AS INTEGER))))," + "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MY_NAME'),'2000-01-01 00:00:00.000000' " + "FROM my_location as legend_persistence_stage) " + "FILES = ('/path/xyz/file1.csv', '/path/xyz/file2.csv') " + @@ -640,7 +646,7 @@ public void testBulkLoadWithDigestAndForceOption() "(\"col_int\", \"col_integer\", \"digest\", \"batch_id\", \"append_time\") " + "FROM " + "(SELECT legend_persistence_stage.$1 as \"col_int\",legend_persistence_stage.$2 as \"col_integer\"," + - "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_int','col_integer'),ARRAY_CONSTRUCT(legend_persistence_stage.$1,legend_persistence_stage.$2))," + + "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_int','col_integer'),ARRAY_CONSTRUCT(CAST(legend_persistence_stage.$1 AS INTEGER),CAST(legend_persistence_stage.$2 AS INTEGER)))," + "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MY_NAME'),'2000-01-01 00:00:00.000000' " + "FROM my_location as legend_persistence_stage) " + "PATTERN = '(/path/xyz/file1.csv)|(/path/xyz/file2.csv)' " + @@ -710,7 +716,7 @@ public void testBulkLoadWithDigestWithFieldsToExcludeAndForceOption() "(\"col_int\", \"col_integer\", \"digest\", \"batch_id\", \"append_time\") " + "FROM " + "(SELECT legend_persistence_stage.$1 as \"col_int\",legend_persistence_stage.$2 as \"col_integer\"," + - "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_integer'),ARRAY_CONSTRUCT(legend_persistence_stage.$2))," + + "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_integer'),ARRAY_CONSTRUCT(CAST(legend_persistence_stage.$2 AS INTEGER)))," + "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MY_NAME'),'2000-01-01 00:00:00.000000' " + "FROM my_location as legend_persistence_stage) " + "PATTERN = '(/path/xyz/file1.csv)|(/path/xyz/file2.csv)' " + @@ -775,7 +781,7 @@ public void testBulkLoadWithDigestAndForceOptionAndOnErrorOption() "(\"col_int\", \"col_integer\", \"digest\", \"batch_id\", \"append_time\") " + "FROM " + "(SELECT legend_persistence_stage.$1 as \"col_int\",legend_persistence_stage.$2 as \"col_integer\"," + - "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_int','col_integer'),ARRAY_CONSTRUCT(legend_persistence_stage.$1,legend_persistence_stage.$2))," + + "LAKEHOUSE_UDF(ARRAY_CONSTRUCT('col_int','col_integer'),ARRAY_CONSTRUCT(CAST(legend_persistence_stage.$1 AS INTEGER),CAST(legend_persistence_stage.$2 AS INTEGER)))," + "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MY_NAME'),'2000-01-01 00:00:00.000000' " + "FROM my_location as legend_persistence_stage) " + "PATTERN = '(/path/xyz/file1.csv)|(/path/xyz/file2.csv)' " +