From 956f460244925a5b783abbee13590b5e996ce616 Mon Sep 17 00:00:00 2001 From: kumuwu Date: Tue, 21 Jan 2025 16:16:26 +0800 Subject: [PATCH 1/4] Implement alter table for duck db --- .../schemaops/statements/AlterTable.java | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java new file mode 100644 index 00000000000..b3bba2c80d6 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java @@ -0,0 +1,113 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.duckdb.sqldom.schemaops.statements; + +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlDomException; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.AlterOperation; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.NotNullColumnConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.Column; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.expresssions.table.Table; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.statements.DDLStatement; + +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.COLUMN; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.DROP; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.WHITE_SPACE; + +public class AlterTable implements DDLStatement +{ + private final AlterOperation operation; + private Table table; + private Column columnToAlter; + + public AlterTable(AlterOperation operation) + { + this.operation = operation; + } + + /* + ALTER TABLE [ IF EXISTS ] tableName ADD COLUMN columnName { columnDefinition } + + ALTER TABLE [ IF EXISTS ] tableName ALTER COLUMN columnName + { columnDefinition } + | { SET NULL } } + */ + @Override + public void genSql(StringBuilder builder) throws SqlDomException + { + validate(); + builder.append(Clause.ALTER.get()); + + builder.append(WHITE_SPACE + Clause.TABLE.get()); + + // Table name + builder.append(WHITE_SPACE); + table.genSqlWithoutAlias(builder); + + // Operation + builder.append(WHITE_SPACE); + if (operation.getParent() == null) + { + builder.append(operation.name()); + } + else + { + builder.append(operation.getParent().name()); + } + builder.append(WHITE_SPACE); + builder.append(COLUMN); + builder.append(WHITE_SPACE); + + // Operation parameters + if (operation == AlterOperation.NULLABLE_COLUMN) + { + columnToAlter.genSqlWithNameOnly(builder); + builder.append(WHITE_SPACE); + builder.append(DROP); + builder.append(WHITE_SPACE); + NotNullColumnConstraint notNullColumnConstraint = new NotNullColumnConstraint(); + notNullColumnConstraint.genSql(builder); + } + else + { + columnToAlter.genSql(builder); + } + } + + @Override + public void push(Object node) + { + if (node instanceof Table) + { + table = (Table) node; + } + else if (node instanceof Column) + { + columnToAlter = (Column) node; + } + } + + void validate() throws SqlDomException + { + if (table == null) + { + throw new SqlDomException("Table is mandatory for Alter Table Command"); + } + if (columnToAlter == null) + { + throw new SqlDomException("Columns details is mandatory for Alter Table Command"); + } + } +} From 480a1827f2b71fc84cb5802cb9df734571809551 Mon Sep 17 00:00:00 2001 From: kumuwu Date: Tue, 21 Jan 2025 16:37:03 +0800 Subject: [PATCH 2/4] Implement alter table for duck db and add tests --- .../relational/duckdb/DuckDBSink.java | 3 + .../duckdb/sql/visitor/AlterVisitor.java | 43 ++++++ .../schemaops/statements/AlterTable.java | 31 +++-- .../sqldom/schemaops/AlterStatementTest.java | 122 ++++++++++++++++++ 4 files changed, 186 insertions(+), 13 deletions(-) create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sql/visitor/AlterVisitor.java create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/AlterStatementTest.java diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/DuckDBSink.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/DuckDBSink.java index 32eb9767c79..091f96b070b 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/DuckDBSink.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/DuckDBSink.java @@ -34,6 +34,7 @@ import org.finos.legend.engine.persistence.components.logicalplan.datasets.StagedFilesDataset; import org.finos.legend.engine.persistence.components.logicalplan.datasets.StagedFilesDatasetReference; import org.finos.legend.engine.persistence.components.logicalplan.datasets.StagedFilesSelection; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Alter; import org.finos.legend.engine.persistence.components.logicalplan.operations.Copy; import org.finos.legend.engine.persistence.components.logicalplan.operations.Update; import org.finos.legend.engine.persistence.components.logicalplan.values.CastFunction; @@ -50,6 +51,7 @@ import org.finos.legend.engine.persistence.components.relational.duckdb.jdbc.DuckDBJdbcHelper; import org.finos.legend.engine.persistence.components.relational.duckdb.sql.DuckDBDataTypeMapping; import org.finos.legend.engine.persistence.components.relational.duckdb.sql.DuckDBJdbcPropertiesToLogicalDataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.duckdb.sql.visitor.AlterVisitor; import org.finos.legend.engine.persistence.components.relational.duckdb.sql.visitor.CastFunctionVisitor; import org.finos.legend.engine.persistence.components.relational.duckdb.sql.visitor.ConcatFunctionVisitor; import org.finos.legend.engine.persistence.components.relational.duckdb.sql.visitor.CopyVisitor; @@ -113,6 +115,7 @@ public class DuckDBSink extends AnsiSqlSink logicalPlanVisitorByClass.put(SchemaDefinition.class, new SchemaDefinitionVisitor()); logicalPlanVisitorByClass.put(Field.class, new FieldVisitor()); logicalPlanVisitorByClass.put(Update.class, new SQLUpdateVisitor()); + logicalPlanVisitorByClass.put(Alter.class, new AlterVisitor()); logicalPlanVisitorByClass.put(ParseJsonFunction.class, new ParseJsonFunctionVisitor()); logicalPlanVisitorByClass.put(Copy.class, new CopyVisitor()); logicalPlanVisitorByClass.put(StagedFilesDatasetReference.class, new StagedFilesDatasetReferenceVisitor()); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sql/visitor/AlterVisitor.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sql/visitor/AlterVisitor.java new file mode 100644 index 00000000000..709c92ee65d --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sql/visitor/AlterVisitor.java @@ -0,0 +1,43 @@ +// Copyright 2025 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.duckdb.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Alter; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.duckdb.sqldom.schemaops.statements.AlterTable; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.AlterOperation; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.ArrayList; +import java.util.List; + +public class AlterVisitor implements LogicalPlanVisitor +{ + + @Override + public VisitorResult visit(PhysicalPlanNode prev, Alter current, VisitorContext context) + { + AlterTable alterTable = new AlterTable(AlterOperation.valueOf(current.operation().name())); + prev.push(alterTable); + + List logicalPlanNodeList = new ArrayList<>(); + logicalPlanNodeList.add(current.dataset()); + logicalPlanNodeList.add(current.columnDetails()); + + return new VisitorResult(alterTable, logicalPlanNodeList); + } +} diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java index b3bba2c80d6..0c23782e5ed 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java @@ -1,4 +1,4 @@ -// Copyright 2024 Goldman Sachs +// Copyright 2025 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -70,19 +70,24 @@ public void genSql(StringBuilder builder) throws SqlDomException builder.append(COLUMN); builder.append(WHITE_SPACE); - // Operation parameters - if (operation == AlterOperation.NULLABLE_COLUMN) + switch (operation) { - columnToAlter.genSqlWithNameOnly(builder); - builder.append(WHITE_SPACE); - builder.append(DROP); - builder.append(WHITE_SPACE); - NotNullColumnConstraint notNullColumnConstraint = new NotNullColumnConstraint(); - notNullColumnConstraint.genSql(builder); - } - else - { - columnToAlter.genSql(builder); + case ADD: + columnToAlter.genSql(builder); + break; + case CHANGE_DATATYPE: + columnToAlter.genSqlWithNameAndTypeOnly(builder); + break; + case NULLABLE_COLUMN: + columnToAlter.genSqlWithNameOnly(builder); + builder.append(WHITE_SPACE); + builder.append(DROP); + builder.append(WHITE_SPACE); + NotNullColumnConstraint notNullColumnConstraint = new NotNullColumnConstraint(); + notNullColumnConstraint.genSql(builder); + break; + default: + throw new SqlDomException("Alter operation " + operation.name() + " not supported"); } } diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/AlterStatementTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/AlterStatementTest.java new file mode 100644 index 00000000000..3c42fdfbbf4 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/AlterStatementTest.java @@ -0,0 +1,122 @@ +// Copyright 2025 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.duckdb.sqldom.schemaops; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Alter; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Operation; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.UpperCaseOptimizer; +import org.finos.legend.engine.persistence.components.relational.duckdb.DuckDBSink; +import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; +import org.finos.legend.engine.persistence.components.transformer.TransformOptions; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static org.finos.legend.engine.persistence.components.logicalplan.operations.BaseTestUtils.*; + +public class AlterStatementTest +{ + public static SchemaDefinition schemaWithAllColumns = SchemaDefinition.builder() + .addFields(colInt) + .addFields(colInteger) + .addFields(colBigint) + .addFields(colTinyint) + .addFields(colSmallint) + .addFields(colChar) + .addFields(colVarchar) + .addFields(colString) + .addFields(colTimestamp) + .addFields(colDatetime) + .addFields(colDate) + .addFields(colReal) + .addFields(colFloat) + .addFields(colDecimal) + .addFields(colDouble) + .addFields(colBinary) + .addFields(colTime) + .addFields(colNumeric) + .addFields(colBoolean) + .addFields(colVarBinary) + .build(); + + @Test + public void testAlterTable() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithAllColumns) + .build(); + Field column = Field.builder().name("column").type(FieldType.of(DataType.VARCHAR, 64, null)).nullable(false).build(); + + Operation add = Alter.of(dataset, Alter.AlterOperation.ADD, column, Optional.empty()); + Operation changeDatatype = Alter.of(dataset, Alter.AlterOperation.CHANGE_DATATYPE, column, Optional.empty()); + Operation nullableColumn = Alter.of(dataset, Alter.AlterOperation.NULLABLE_COLUMN, column, Optional.empty()); + + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(add, changeDatatype, nullableColumn).build(); + RelationalTransformer transformer = new RelationalTransformer(DuckDBSink.get()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + + String expectedAdd = "ALTER TABLE \"my_db\".\"my_schema\".\"my_table\" ADD COLUMN \"column\" VARCHAR NOT NULL"; + String expectedChangeDataType = "ALTER TABLE \"my_db\".\"my_schema\".\"my_table\" ALTER COLUMN \"column\" VARCHAR"; + String expectedNullableColumn = "ALTER TABLE \"my_db\".\"my_schema\".\"my_table\" ALTER COLUMN \"column\" DROP NOT NULL"; + + Assertions.assertEquals(expectedAdd, list.get(0)); + Assertions.assertEquals(expectedChangeDataType, list.get(1)); + Assertions.assertEquals(expectedNullableColumn, list.get(2)); + } + + @Test + public void testAlterTableWithUpperCase() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithAllColumns) + .build(); + Field column = Field.builder().name("column").type(FieldType.of(DataType.VARCHAR, 64, null)).nullable(false).build(); + + Operation add = Alter.of(dataset, Alter.AlterOperation.ADD, column, Optional.empty()); + Operation changeDatatype = Alter.of(dataset, Alter.AlterOperation.CHANGE_DATATYPE, column, Optional.empty()); + Operation nullableColumn = Alter.of(dataset, Alter.AlterOperation.NULLABLE_COLUMN, column, Optional.empty()); + + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(add, changeDatatype, nullableColumn).build(); + RelationalTransformer transformer = new RelationalTransformer(DuckDBSink.get(), TransformOptions.builder().addOptimizers(new UpperCaseOptimizer()).build()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + + String expectedAdd = "ALTER TABLE \"MY_DB\".\"MY_SCHEMA\".\"MY_TABLE\" ADD COLUMN \"COLUMN\" VARCHAR NOT NULL"; + String expectedChangeDataType = "ALTER TABLE \"MY_DB\".\"MY_SCHEMA\".\"MY_TABLE\" ALTER COLUMN \"COLUMN\" VARCHAR"; + String expectedNullableColumn = "ALTER TABLE \"MY_DB\".\"MY_SCHEMA\".\"MY_TABLE\" ALTER COLUMN \"COLUMN\" DROP NOT NULL"; + + Assertions.assertEquals(expectedAdd, list.get(0)); + Assertions.assertEquals(expectedChangeDataType, list.get(1)); + Assertions.assertEquals(expectedNullableColumn, list.get(2)); + } +} From aeb86ec59f147cb391c0c9a75a78d7b23ed0899d Mon Sep 17 00:00:00 2001 From: kumuwu Date: Tue, 21 Jan 2025 17:58:57 +0800 Subject: [PATCH 3/4] Fix change data type and add tests --- .../schemaops/statements/AlterTable.java | 10 +- .../persistence/components/e2e/AlterTest.java | 155 ++++++++++++++++++ .../persistence/components/e2e/TestUtils.java | 4 +- .../sqldom/schemaops/AlterStatementTest.java | 4 +- .../expected/add_expected_pass.csv | 3 + .../expected/change_type_expected_pass.csv | 3 + .../nullable_column_expected_pass.csv | 3 + .../data/alter-table/input/add_data_pass.csv | 3 + .../input/change_type_data_pass.csv | 3 + .../input/nullable_column_data_pass.csv | 3 + 10 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/e2e/AlterTest.java create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/add_expected_pass.csv create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/change_type_expected_pass.csv create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/nullable_column_expected_pass.csv create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/add_data_pass.csv create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/change_type_data_pass.csv create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/nullable_column_data_pass.csv diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java index 0c23782e5ed..4f817b96c6e 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/main/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/statements/AlterTable.java @@ -23,7 +23,9 @@ import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.statements.DDLStatement; import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.COLUMN; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.DATA_TYPE; import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.DROP; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.SET; import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.WHITE_SPACE; public class AlterTable implements DDLStatement @@ -76,7 +78,13 @@ public void genSql(StringBuilder builder) throws SqlDomException columnToAlter.genSql(builder); break; case CHANGE_DATATYPE: - columnToAlter.genSqlWithNameAndTypeOnly(builder); + columnToAlter.genSqlWithNameOnly(builder); + builder.append(WHITE_SPACE); + builder.append(SET.get()); + builder.append(WHITE_SPACE); + builder.append(DATA_TYPE.get()); + builder.append(WHITE_SPACE); + columnToAlter.genSqlWithTypeOnly(builder); break; case NULLABLE_COLUMN: columnToAlter.genSqlWithNameOnly(builder); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/e2e/AlterTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/e2e/AlterTest.java new file mode 100644 index 00000000000..f1ddf3cf3be --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/e2e/AlterTest.java @@ -0,0 +1,155 @@ +// Copyright 2025 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanFactory; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Alter; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Operation; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.duckdb.DuckDBSink; +import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.alterColumn; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.alterColumnName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.batchIdName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.digestName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.expiryDateName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.idName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.incomeChanged; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.incomeName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.mainTableName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.name; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.nameName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.startTimeName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.testDatabaseName; +import static org.finos.legend.engine.persistence.components.e2e.TestUtils.testSchemaName; + +class AlterTest extends BaseTest +{ + private String basePath = "src/test/resources/data/alter-table/"; + + @Test + void testAlterTableAddColumn() throws Exception + { + // Prepare main table + DatasetDefinition mainTable = TestUtils.getBasicMainTable(); + + RelationalTransformer transformer = new RelationalTransformer(DuckDBSink.get()); + LogicalPlan tableCreationPlan = LogicalPlanFactory.getDatasetCreationPlan(mainTable, true); + SqlPlan tableCreationPhysicalPlan = transformer.generatePhysicalPlan(tableCreationPlan); + executor.executePhysicalPlan(tableCreationPhysicalPlan); + String inputPath = basePath + "input/add_data_pass.csv"; + insertMainData(inputPath); + + // Generate and execute schema evolution plan + Operation add = Alter.of(mainTable, Alter.AlterOperation.ADD, alterColumn, Optional.empty()); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(add).build(); + SqlPlan schemaEvolutionPhysicalPlan = transformer.generatePhysicalPlan(logicalPlan); + executor.executePhysicalPlan(schemaEvolutionPhysicalPlan); + + // Verify the new schema + List> actualTableData = duckDBSink.executeQuery("select * from \"TEST\".\"main\""); + List expectedNewSchema = Arrays.asList(idName, nameName, incomeName, startTimeName, expiryDateName, digestName, batchIdName, alterColumnName); + String expectedPath = basePath + "expected/add_expected_pass.csv"; + TestUtils.assertTableColumnsEquals(expectedNewSchema, actualTableData); + TestUtils.assertFileAndTableDataEquals(expectedNewSchema.toArray(new String[]{}), expectedPath, actualTableData); + } + + @Test + void testAlterTableChangeDataType() throws Exception + { + // Prepare main table + DatasetDefinition mainTable = TestUtils.getBasicMainTable(); + + RelationalTransformer transformer = new RelationalTransformer(DuckDBSink.get()); + LogicalPlan tableCreationPlan = LogicalPlanFactory.getDatasetCreationPlan(mainTable, true); + SqlPlan tableCreationPhysicalPlan = transformer.generatePhysicalPlan(tableCreationPlan); + executor.executePhysicalPlan(tableCreationPhysicalPlan); + String inputPath = basePath + "input/change_type_data_pass.csv"; + insertMainData(inputPath); + + // Assert column is of type BIGINT before operation + String dataType = TestUtils.getColumnDataTypeFromTable(duckDBSink.connection(), testDatabaseName, testSchemaName, mainTableName, incomeName); + Assertions.assertEquals("BIGINT", dataType); + + // Generate and execute schema evolution plan + Operation changeDataType = Alter.of(mainTable, Alter.AlterOperation.CHANGE_DATATYPE, incomeChanged, Optional.empty()); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(changeDataType).build(); + SqlPlan schemaEvolutionPhysicalPlan = transformer.generatePhysicalPlan(logicalPlan); + executor.executePhysicalPlan(schemaEvolutionPhysicalPlan); + + // Verify the new schema + List> actualTableData = duckDBSink.executeQuery("select * from \"TEST\".\"main\""); + List expectedNewSchema = Arrays.asList(idName, nameName, incomeName, startTimeName, expiryDateName, digestName, batchIdName); + String expectedPath = basePath + "expected/change_type_expected_pass.csv"; + TestUtils.assertTableColumnsEquals(expectedNewSchema, actualTableData); + TestUtils.assertFileAndTableDataEquals(expectedNewSchema.toArray(new String[]{}), expectedPath, actualTableData); + + // Assert column is of type INTEGER after operation + dataType = TestUtils.getColumnDataTypeFromTable(duckDBSink.connection(), testDatabaseName, testSchemaName, mainTableName, incomeName); + Assertions.assertEquals("INTEGER", dataType); + } + + @Test + void testAlterTableNullableColumn() throws Exception + { + // Prepare main table + DatasetDefinition mainTable = TestUtils.getBasicMainTable(); + LogicalPlan tableCreationPlan = LogicalPlanFactory.getDatasetCreationPlan(mainTable, true); + + RelationalTransformer transformer = new RelationalTransformer(DuckDBSink.get()); + SqlPlan tableCreationPhysicalPlan = transformer.generatePhysicalPlan(tableCreationPlan); + executor.executePhysicalPlan(tableCreationPhysicalPlan); + String inputPath = basePath + "input/nullable_column_data_pass.csv"; + insertMainData(inputPath); + + // Assert column is not nullable before operation + Assertions.assertEquals("NO", TestUtils.getIsColumnNullableFromTable(duckDBSink, mainTableName, nameName)); + + // Generate and execute schema evolution plan + Operation nullableColumn = Alter.of(mainTable, Alter.AlterOperation.NULLABLE_COLUMN, name, Optional.empty()); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(nullableColumn).build(); + SqlPlan schemaEvolutionPhysicalPlan = transformer.generatePhysicalPlan(logicalPlan); + executor.executePhysicalPlan(schemaEvolutionPhysicalPlan); + + // Verify the new schema + List> actualTableData = duckDBSink.executeQuery("select * from \"TEST\".\"main\""); + List expectedNewSchema = Arrays.asList(idName, nameName, incomeName, startTimeName, expiryDateName, digestName, batchIdName); + String expectedPath = basePath + "expected/nullable_column_expected_pass.csv"; + TestUtils.assertTableColumnsEquals(expectedNewSchema, actualTableData); + TestUtils.assertFileAndTableDataEquals(expectedNewSchema.toArray(new String[]{}), expectedPath, actualTableData); + + // Assert column is nullable after operation + Assertions.assertEquals("YES", TestUtils.getIsColumnNullableFromTable(duckDBSink, mainTableName, nameName)); + } + + private void insertMainData(String path) throws Exception + { + String loadSql = "TRUNCATE TABLE \"TEST\".\"main\";" + + "COPY \"TEST\".\"main\"" + + "(\"id\", \"name\", \"income\", \"start_time\", \"expiry_date\", \"digest\", \"batch_id\")" + + " FROM '" + path + "' CSV"; + duckDBSink.executeStatement(loadSql); + } +} \ No newline at end of file diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/e2e/TestUtils.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/e2e/TestUtils.java index 65bbdaa51ca..4cf9442dea0 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/e2e/TestUtils.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/e2e/TestUtils.java @@ -1512,8 +1512,8 @@ public static List getColumnsFromTable(Connection connection, String dat // This is to check the actual database table - whether columns have the right nullability public static String getIsColumnNullableFromTable(RelationalExecutionHelper sink, String tableName, String columnName) { - List> result = sink.executeQuery("SELECT IS_NULLABLE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='" + tableName + "' and COLUMN_NAME ='" + columnName + "'"); - return result.get(0).get("IS_NULLABLE").toString(); + List> result = sink.executeQuery("SELECT is_nullable FROM information_schema.columns WHERE table_name ='" + tableName + "' and column_name ='" + columnName + "'"); + return result.get(0).get("is_nullable").toString(); } // This is to check the actual database table - the length (precision) of the column data type diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/AlterStatementTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/AlterStatementTest.java index 3c42fdfbbf4..02eb2d9c967 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/AlterStatementTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/java/org/finos/legend/engine/persistence/components/relational/duckdb/sqldom/schemaops/AlterStatementTest.java @@ -82,7 +82,7 @@ public void testAlterTable() List list = physicalPlan.getSqlList(); String expectedAdd = "ALTER TABLE \"my_db\".\"my_schema\".\"my_table\" ADD COLUMN \"column\" VARCHAR NOT NULL"; - String expectedChangeDataType = "ALTER TABLE \"my_db\".\"my_schema\".\"my_table\" ALTER COLUMN \"column\" VARCHAR"; + String expectedChangeDataType = "ALTER TABLE \"my_db\".\"my_schema\".\"my_table\" ALTER COLUMN \"column\" SET DATA TYPE VARCHAR"; String expectedNullableColumn = "ALTER TABLE \"my_db\".\"my_schema\".\"my_table\" ALTER COLUMN \"column\" DROP NOT NULL"; Assertions.assertEquals(expectedAdd, list.get(0)); @@ -112,7 +112,7 @@ public void testAlterTableWithUpperCase() List list = physicalPlan.getSqlList(); String expectedAdd = "ALTER TABLE \"MY_DB\".\"MY_SCHEMA\".\"MY_TABLE\" ADD COLUMN \"COLUMN\" VARCHAR NOT NULL"; - String expectedChangeDataType = "ALTER TABLE \"MY_DB\".\"MY_SCHEMA\".\"MY_TABLE\" ALTER COLUMN \"COLUMN\" VARCHAR"; + String expectedChangeDataType = "ALTER TABLE \"MY_DB\".\"MY_SCHEMA\".\"MY_TABLE\" ALTER COLUMN \"COLUMN\" SET DATA TYPE VARCHAR"; String expectedNullableColumn = "ALTER TABLE \"MY_DB\".\"MY_SCHEMA\".\"MY_TABLE\" ALTER COLUMN \"COLUMN\" DROP NOT NULL"; Assertions.assertEquals(expectedAdd, list.get(0)); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/add_expected_pass.csv b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/add_expected_pass.csv new file mode 100644 index 00000000000..da7b7403265 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/add_expected_pass.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2020-01-01 00:00:00.0,2022-12-01,DIGEST1,1,null +2,ROBERT,2000,2020-01-02 00:00:00.0,2022-12-02,DIGEST2,1,null +3,ANDY,3000,2020-01-03 00:00:00.0,2022-12-03,DIGEST3,1,null diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/change_type_expected_pass.csv b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/change_type_expected_pass.csv new file mode 100644 index 00000000000..71e854be809 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/change_type_expected_pass.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2020-01-01 00:00:00.0,2022-12-01,DIGEST1,1 +2,ROBERT,2000,2020-01-02 00:00:00.0,2022-12-02,DIGEST2,1 +3,ANDY,3000,2020-01-03 00:00:00.0,2022-12-03,DIGEST3,1 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/nullable_column_expected_pass.csv b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/nullable_column_expected_pass.csv new file mode 100644 index 00000000000..71e854be809 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/expected/nullable_column_expected_pass.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2020-01-01 00:00:00.0,2022-12-01,DIGEST1,1 +2,ROBERT,2000,2020-01-02 00:00:00.0,2022-12-02,DIGEST2,1 +3,ANDY,3000,2020-01-03 00:00:00.0,2022-12-03,DIGEST3,1 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/add_data_pass.csv b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/add_data_pass.csv new file mode 100644 index 00000000000..71e854be809 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/add_data_pass.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2020-01-01 00:00:00.0,2022-12-01,DIGEST1,1 +2,ROBERT,2000,2020-01-02 00:00:00.0,2022-12-02,DIGEST2,1 +3,ANDY,3000,2020-01-03 00:00:00.0,2022-12-03,DIGEST3,1 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/change_type_data_pass.csv b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/change_type_data_pass.csv new file mode 100644 index 00000000000..71e854be809 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/change_type_data_pass.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2020-01-01 00:00:00.0,2022-12-01,DIGEST1,1 +2,ROBERT,2000,2020-01-02 00:00:00.0,2022-12-02,DIGEST2,1 +3,ANDY,3000,2020-01-03 00:00:00.0,2022-12-03,DIGEST3,1 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/nullable_column_data_pass.csv b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/nullable_column_data_pass.csv new file mode 100644 index 00000000000..71e854be809 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-duckdb/src/test/resources/data/alter-table/input/nullable_column_data_pass.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2020-01-01 00:00:00.0,2022-12-01,DIGEST1,1 +2,ROBERT,2000,2020-01-02 00:00:00.0,2022-12-02,DIGEST2,1 +3,ANDY,3000,2020-01-03 00:00:00.0,2022-12-03,DIGEST3,1 From 5eb52b7689045c1d57457440d6515d5fbed68502 Mon Sep 17 00:00:00 2001 From: Mythreyi Date: Thu, 23 Jan 2025 09:24:57 +0800 Subject: [PATCH 4/4] Persistence Component: Fixing bug in IngestModeOptimizer for delete strategy in partitioning with test --- .../relational/api/IngestModeOptimizer.java | 4 +- .../unitemporal/UnitemporalSnapshotTest.java | 66 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/IngestModeOptimizer.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/IngestModeOptimizer.java index 1ef76be8459..4c0f696b210 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/IngestModeOptimizer.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/IngestModeOptimizer.java @@ -87,7 +87,9 @@ public IngestMode visitUnitemporalSnapshot(UnitemporalSnapshotAbstract unitempor .addAllPartitionFields(partition.partitionFields()) .addAllPartitionSpecList(derivePartitionSpecList(partition.partitionFields(), partition.maxPartitionSpecFilters())) .derivePartitionSpec(partition.derivePartitionSpec()) - .maxPartitionSpecFilters(partition.maxPartitionSpecFilters()).build()) + .maxPartitionSpecFilters(partition.maxPartitionSpecFilters()) + .deleteStrategy(partition.deleteStrategy()) + .build()) .build(); } } diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotTest.java index 6e504d07e4d..280470b6218 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotTest.java @@ -822,6 +822,72 @@ void testUnitemporalSnapshotMilestoningLogicWithPartitionSpecDeleteAll() throws executePlansAndVerifyResults(ingestMode, options, datasets, schema, expectedDataPass4, expectedStats, fixedClock_2000_01_01); } + /* + Scenario: Test milestoning Logic with Partition delete all with derive partitio spec when staging table pre populated + Empty Batch Handling : Default + */ + + @Test + void testUnitemporalSnapshotMilestoningLogicWithDerivePartitionSpecDeleteAll() throws Exception + { + DatasetDefinition mainTable = TestUtils.getEntityPriceMainTableWithoutDigest(); + DatasetDefinition stagingTable = TestUtils.getEntityPriceStagingTableWithoutDigest(); + + String[] schema = new String[]{dateName, entityName, priceName, volumeName, batchIdInName, batchIdOutName, batchTimeInName, batchTimeOutName}; + + // Create staging table + createStagingTable(stagingTable); + + Partitioning partitioning = Partitioning.builder() + .addAllPartitionFields(Collections.singletonList(dateName)) + .derivePartitionSpec(true) + .deleteStrategy(DeleteAllStrategy.builder().build()) + .build(); + + UnitemporalSnapshot ingestMode = UnitemporalSnapshot.builder() + .transactionMilestoning(BatchIdAndDateTime.builder() + .batchIdInName(batchIdInName) + .batchIdOutName(batchIdOutName) + .dateTimeInName(batchTimeInName) + .dateTimeOutName(batchTimeOutName) + .build()) + .partitioningStrategy(partitioning) + .build(); + + PlannerOptions options = PlannerOptions.builder().collectStatistics(true).build(); + Datasets datasets = Datasets.of(mainTable, stagingTable); + + // ------------ Perform unitemporal snapshot milestoning Pass1 ------------------------ + String dataPass1 = basePathForInput + "with_partition/no_version_delete_all/staging_data_pass1.csv"; + String expectedDataPass1 = basePathForExpected + "with_partition/no_version_delete_all/expected_pass1.csv"; + // 1. Load staging table + loadStagingDataForWithPartitionWithoutDigest(dataPass1); + // 2. Execute plans and verify results + Map expectedStats = createExpectedStatsMap(6, 0, 6, 0, 0); + executePlansAndVerifyResults(ingestMode, options, datasets, schema, expectedDataPass1, expectedStats, fixedClock_2000_01_01); + + // ------------ Perform unitemporal snapshot milestoning Pass2 ------------------------ + + String dataPass2 = basePathForInput + "with_partition/no_version_delete_all/staging_data_pass2.csv"; + String expectedDataPass2 = basePathForExpected + "with_partition/no_version_delete_all/expected_pass2.csv"; + // 1. Load staging table + loadStagingDataForWithPartitionWithoutDigest(dataPass2); + // 2. Execute plans and verify results + expectedStats = createExpectedStatsMap(3, 0, 1, 2, 1); + executePlansAndVerifyResults(ingestMode, options, datasets, schema, expectedDataPass2, expectedStats, fixedClock_2000_01_01); + + // ------------ Perform unitemporal snapshot milestoning Pass3 (Empty Batch) ------------------------ + options = options.withCleanupStagingData(true); + String dataPass3 = "src/test/resources/data/empty_file.csv"; + String expectedDataPass3 = basePathForExpected + "with_partition/no_version_delete_all/expected_pass3.csv"; + // 1. Load Staging table + loadStagingDataForWithPartitionWithoutDigest(dataPass3); + // 2. Execute plans and verify results + expectedStats = createExpectedStatsMap(0, 0, 0, 0, 0); + executePlansAndVerifyResults(ingestMode, options, datasets, schema, expectedDataPass3, expectedStats, fixedClock_2000_01_01); + + } + /* Scenario: Test milestoning Logic with max version and with Partition delete all when staging table pre populated with upper case */