From f78869f82a038458cb233263c1cd7ae69ad4cf79 Mon Sep 17 00:00:00 2001 From: prasar-ashutosh Date: Fri, 3 May 2024 16:13:56 +0530 Subject: [PATCH] Fix the tests --- .../components/util/LogicalPlanUtils.java | 2 +- .../components/AnsiTestArtifacts.java | 4 +-- .../UnitemporalSnapshotBatchIdBasedTest.java | 30 +++++++++---------- .../ingestmode/BigQueryTestArtifacts.java | 4 +-- .../UnitemporalSnapshotBatchIdBasedTest.java | 26 ++++++++-------- .../ingestmode/MemsqlTestArtifacts.java | 4 +-- .../UnitemporalSnapshotBatchIdBasedTest.java | 30 +++++++++---------- .../components/PostgresTestArtifacts.java | 4 +-- .../UnitemporalSnapshotBatchIdBasedTest.java | 30 +++++++++---------- .../persistence/components/BaseTest.java | 14 ++++----- 10 files changed, 74 insertions(+), 74 deletions(-) diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtils.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtils.java index 84b602203af..b258d76e704 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtils.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtils.java @@ -144,7 +144,7 @@ public static Condition getPartitionSpecMatchCondition(Dataset dataSet, List Equals.of( FieldValue.builder().datasetRef(dataSet.datasetReference()).fieldName(columnValuePair.getKey()).build(), - StringValue.of((String) columnValuePair.getValue())) + columnValuePair.getValue() instanceof Number ? ObjectValue.of(columnValuePair.getValue()) : StringValue.of((String) columnValuePair.getValue())) ) .collect(Collectors.toList())) ).collect(Collectors.toList())); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/AnsiTestArtifacts.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/AnsiTestArtifacts.java index 07e51009668..79792688f90 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/AnsiTestArtifacts.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/AnsiTestArtifacts.java @@ -64,7 +64,7 @@ public class AnsiTestArtifacts "\"batch_id_in\" INTEGER NOT NULL,\"batch_id_out\" INTEGER,PRIMARY KEY (\"id\", \"name\", \"batch_id_in\"))"; public static String expectedMainTableWithMultiPartitionsCreateQuery = "CREATE TABLE IF NOT EXISTS \"mydb\".\"main\"(" + - "\"id\" INTEGER NOT NULL,\"name\" VARCHAR NOT NULL,\"amount\" DOUBLE,\"account_type\" VARCHAR,\"biz_date\" DATE,\"digest\" VARCHAR," + + "\"id\" INTEGER NOT NULL,\"name\" VARCHAR NOT NULL,\"amount\" DOUBLE,\"account_type\" INTEGER,\"biz_date\" DATE,\"digest\" VARCHAR," + "\"batch_id_in\" INTEGER NOT NULL,\"batch_id_out\" INTEGER,PRIMARY KEY (\"id\", \"name\", \"batch_id_in\"))"; public static String expectedMainTableBatchIdAndVersionBasedCreateQuery = "CREATE TABLE IF NOT EXISTS \"mydb\".\"main\"(" + @@ -80,7 +80,7 @@ public class AnsiTestArtifacts "\"BATCH_ID_IN\" INTEGER NOT NULL,\"BATCH_ID_OUT\" INTEGER,PRIMARY KEY (\"ID\", \"NAME\", \"BATCH_ID_IN\"))"; public static String expectedMainTableMultiPartitionCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS \"MYDB\".\"MAIN\"" + - "(\"ID\" INTEGER NOT NULL,\"NAME\" VARCHAR NOT NULL,\"AMOUNT\" DOUBLE,\"ACCOUNT_TYPE\" VARCHAR,\"BIZ_DATE\" DATE,\"DIGEST\" VARCHAR," + + "(\"ID\" INTEGER NOT NULL,\"NAME\" VARCHAR NOT NULL,\"AMOUNT\" DOUBLE,\"ACCOUNT_TYPE\" INTEGER,\"BIZ_DATE\" DATE,\"DIGEST\" VARCHAR," + "\"BATCH_ID_IN\" INTEGER NOT NULL,\"BATCH_ID_OUT\" INTEGER,PRIMARY KEY (\"ID\", \"NAME\", \"BATCH_ID_IN\"))"; public static String expectedMainTableTimeBasedCreateQuery = "CREATE TABLE IF NOT EXISTS \"mydb\".\"main\"(" + diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotBatchIdBasedTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotBatchIdBasedTest.java index 25a6527c7f9..7c8afbbe70d 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotBatchIdBasedTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotBatchIdBasedTest.java @@ -219,9 +219,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener "WHERE (sink.\"batch_id_out\" = 999999999) AND " + "(NOT (EXISTS (SELECT * FROM \"mydb\".\"staging\" as stage " + "WHERE ((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (sink.\"digest\" = stage.\"digest\")))) " + - "AND (((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2')))"; + "AND (((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2)))"; String expectedUpsertQuery = "INSERT INTO \"mydb\".\"main\" " + "(\"id\", \"name\", \"amount\", \"account_type\", \"biz_date\", \"digest\", \"batch_id_in\", \"batch_id_out\") " + @@ -230,9 +230,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener "FROM \"mydb\".\"staging\" as stage WHERE " + "NOT (stage.\"digest\" IN (SELECT sink.\"digest\" FROM \"mydb\".\"main\" as sink WHERE " + "(sink.\"batch_id_out\" = 999999999) AND " + - "(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2'))))))"; + "(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2))))))"; Assertions.assertEquals(AnsiTestArtifacts.expectedMainTableWithMultiPartitionsCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(getExpectedMetadataTableCreateQuery(), preActionsSql.get(1)); @@ -254,9 +254,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe "WHERE (sink.\"BATCH_ID_OUT\" = 999999999) AND " + "(NOT (EXISTS (SELECT * FROM \"MYDB\".\"STAGING\" as stage " + "WHERE ((sink.\"ID\" = stage.\"ID\") AND (sink.\"NAME\" = stage.\"NAME\")) AND (sink.\"DIGEST\" = stage.\"DIGEST\")))) " + - "AND (((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-01')) " + - "OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-02')) " + - "OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_2') AND (sink.\"BIZ_DATE\" = '2024-01-02')))"; + "AND (((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-01')) " + + "OR ((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-02')) " + + "OR ((sink.\"ACCOUNT_TYPE\" = 2) AND (sink.\"BIZ_DATE\" = '2024-01-02')))"; String expectedUpsertQuery = "INSERT INTO \"MYDB\".\"MAIN\" " + "(\"ID\", \"NAME\", \"AMOUNT\", \"ACCOUNT_TYPE\", \"BIZ_DATE\", \"DIGEST\", \"BATCH_ID_IN\", \"BATCH_ID_OUT\") " + @@ -264,9 +264,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe "(SELECT COALESCE(MAX(BATCH_METADATA.\"TABLE_BATCH_ID\"),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.\"TABLE_NAME\") = 'MAIN')," + "999999999 FROM \"MYDB\".\"STAGING\" as stage " + "WHERE NOT (stage.\"DIGEST\" IN (SELECT sink.\"DIGEST\" FROM \"MYDB\".\"MAIN\" as sink WHERE (sink.\"BATCH_ID_OUT\" = 999999999) " + - "AND (((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-01')) " + - "OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-02')) " + - "OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_2') AND (sink.\"BIZ_DATE\" = '2024-01-02'))))))"; + "AND (((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-01')) " + + "OR ((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-02')) " + + "OR ((sink.\"ACCOUNT_TYPE\" = 2) AND (sink.\"BIZ_DATE\" = '2024-01-02'))))))"; Assertions.assertEquals(AnsiTestArtifacts.expectedMainTableMultiPartitionCreateQueryWithUpperCase, preActionsSql.get(0)); Assertions.assertEquals(getExpectedMetadataTableCreateQueryWithUpperCase(), preActionsSql.get(1)); @@ -285,9 +285,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListWithEmptyBatchHandling String expectedMilestoneQuery = "UPDATE \"mydb\".\"main\" as sink SET " + "sink.\"batch_id_out\" = (SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')-1 " + "WHERE (sink.\"batch_id_out\" = 999999999) AND " + - "(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2')))"; + "(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2)))"; Assertions.assertEquals(AnsiTestArtifacts.expectedMainTableWithMultiPartitionsCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(getExpectedMetadataTableCreateQuery(), preActionsSql.get(1)); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BigQueryTestArtifacts.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BigQueryTestArtifacts.java index 721d0076c50..061e069cf30 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BigQueryTestArtifacts.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BigQueryTestArtifacts.java @@ -202,11 +202,11 @@ public class BigQueryTestArtifacts "`batch_id_in` INT64 NOT NULL,`batch_id_out` INT64,PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)"; public static String expectedMainTableWithMultiPartitionCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + - "`id` INT64 NOT NULL,`name` STRING NOT NULL,`amount` FLOAT64,`account_type` STRING,`biz_date` DATE,`digest` STRING," + + "`id` INT64 NOT NULL,`name` STRING NOT NULL,`amount` FLOAT64,`account_type` INT64,`biz_date` DATE,`digest` STRING," + "`batch_id_in` INT64 NOT NULL,`batch_id_out` INT64,PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)"; public static String expectedMainTableWithMultiPartitionCreateQueryUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + - "(`ID` INT64 NOT NULL,`NAME` STRING NOT NULL,`AMOUNT` FLOAT64,`ACCOUNT_TYPE` STRING,`BIZ_DATE` DATE,`DIGEST` STRING," + + "(`ID` INT64 NOT NULL,`NAME` STRING NOT NULL,`AMOUNT` FLOAT64,`ACCOUNT_TYPE` INT64,`BIZ_DATE` DATE,`DIGEST` STRING," + "`BATCH_ID_IN` INT64 NOT NULL,`BATCH_ID_OUT` INT64,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`) NOT ENFORCED)"; public static String expectedMetadataTableCreateQuery = "CREATE TABLE IF NOT EXISTS batch_metadata" + diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java index 0c99ac890ca..52ea588eea9 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java @@ -194,9 +194,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener "WHERE (sink.`batch_id_out` = 999999999) AND " + "(NOT (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE " + "((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) AND " + - "(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2')))"; + "(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2)))"; String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + "(`id`, `name`, `amount`, `account_type`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + @@ -205,9 +205,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + "FROM `mydb`.`staging` as stage WHERE " + "NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND " + - "(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2'))))))"; + "(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2))))))"; Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableWithMultiPartitionCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); @@ -228,9 +228,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe "sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1 " + "WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (NOT (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage " + "WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) AND (sink.`DIGEST` = stage.`DIGEST`)))) " + - "AND (((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-01')) " + - "OR ((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-02')) " + - "OR ((sink.`ACCOUNT_TYPE` = 'TYPE_2') AND (sink.`BIZ_DATE` = '2024-01-02')))"; + "AND (((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-01')) " + + "OR ((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-02')) " + + "OR ((sink.`ACCOUNT_TYPE` = 2) AND (sink.`BIZ_DATE` = '2024-01-02')))"; String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` " + "(`ID`, `NAME`, `AMOUNT`, `ACCOUNT_TYPE`, `BIZ_DATE`, `DIGEST`, `BATCH_ID_IN`, `BATCH_ID_OUT`) " + @@ -238,9 +238,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe "(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')," + "999999999 FROM `MYDB`.`STAGING` as stage WHERE " + "NOT (stage.`DIGEST` IN (SELECT sink.`DIGEST` FROM `MYDB`.`MAIN` as sink WHERE (sink.`BATCH_ID_OUT` = 999999999) AND " + - "(((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-01')) " + - "OR ((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-02')) " + - "OR ((sink.`ACCOUNT_TYPE` = 'TYPE_2') AND (sink.`BIZ_DATE` = '2024-01-02'))))))"; + "(((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-01')) " + + "OR ((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-02')) " + + "OR ((sink.`ACCOUNT_TYPE` = 2) AND (sink.`BIZ_DATE` = '2024-01-02'))))))"; Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableWithMultiPartitionCreateQueryUpperCase, preActionsSql.get(0)); Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); @@ -256,7 +256,7 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListWithEmptyBatchHandling List milestoningSql = operations.ingestSql(); List metadataIngestSql = operations.metadataIngestSql(); - String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 WHERE (sink.`batch_id_out` = 999999999) AND (((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2')))"; + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 WHERE (sink.`batch_id_out` = 999999999) AND (((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2)))"; Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableWithMultiPartitionCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/MemsqlTestArtifacts.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/MemsqlTestArtifacts.java index 3bc013700ab..5cd8c30b403 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/MemsqlTestArtifacts.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/MemsqlTestArtifacts.java @@ -193,7 +193,7 @@ public class MemsqlTestArtifacts "`batch_id_in` INTEGER NOT NULL,`batch_id_out` INTEGER,PRIMARY KEY (`id`, `name`, `batch_id_in`))"; public static String expectedMainTableMultiPartitionKeysCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + - "`id` INTEGER NOT NULL,`name` VARCHAR(256) NOT NULL,`amount` DOUBLE,`account_type` VARCHAR(256),`biz_date` DATE,`digest` VARCHAR(256)," + + "`id` INTEGER NOT NULL,`name` VARCHAR(256) NOT NULL,`amount` DOUBLE,`account_type` INTEGER,`biz_date` DATE,`digest` VARCHAR(256)," + "`batch_id_in` INTEGER NOT NULL,`batch_id_out` INTEGER,PRIMARY KEY (`id`, `name`, `batch_id_in`))"; public static String expectedMetadataTableCreateQuery = "CREATE TABLE IF NOT EXISTS batch_metadata" + @@ -219,7 +219,7 @@ public class MemsqlTestArtifacts "`BATCH_ID_IN` INTEGER NOT NULL,`BATCH_ID_OUT` INTEGER,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`))"; public static String expectedMainTableMultiPartitionsCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + - "(`ID` INTEGER NOT NULL,`NAME` VARCHAR(256) NOT NULL,`AMOUNT` DOUBLE,`ACCOUNT_TYPE` VARCHAR(256),`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," + + "(`ID` INTEGER NOT NULL,`NAME` VARCHAR(256) NOT NULL,`AMOUNT` DOUBLE,`ACCOUNT_TYPE` INTEGER,`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," + "`BATCH_ID_IN` INTEGER NOT NULL,`BATCH_ID_OUT` INTEGER,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`))"; public static String expectedMetadataTableIngestQuery = "INSERT INTO batch_metadata (`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`)" + diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java index 29743d9d76b..efa5a82352a 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java @@ -194,9 +194,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener "sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + "WHERE (sink.`batch_id_out` = 999999999) AND " + "(NOT (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) " + - "AND (((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2')))"; + "AND (((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2)))"; String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + "(`id`, `name`, `amount`, `account_type`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + @@ -205,9 +205,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener "FROM `mydb`.`staging` as stage WHERE " + "NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE " + "(sink.`batch_id_out` = 999999999) AND " + - "(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2'))))))"; + "(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2))))))"; Assertions.assertEquals(MemsqlTestArtifacts.expectedMainTableMultiPartitionKeysCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(MemsqlTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); @@ -228,9 +228,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe "sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1 " + "WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (NOT (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage " + "WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) AND (sink.`DIGEST` = stage.`DIGEST`)))) " + - "AND (((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-01')) " + - "OR ((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-02')) " + - "OR ((sink.`ACCOUNT_TYPE` = 'TYPE_2') AND (sink.`BIZ_DATE` = '2024-01-02')))"; + "AND (((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-01')) " + + "OR ((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-02')) " + + "OR ((sink.`ACCOUNT_TYPE` = 2) AND (sink.`BIZ_DATE` = '2024-01-02')))"; String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` " + "(`ID`, `NAME`, `AMOUNT`, `ACCOUNT_TYPE`, `BIZ_DATE`, `DIGEST`, `BATCH_ID_IN`, `BATCH_ID_OUT`) " + @@ -238,9 +238,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe "(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')," + "999999999 FROM `MYDB`.`STAGING` as stage WHERE " + "NOT (stage.`DIGEST` IN (SELECT sink.`DIGEST` FROM `MYDB`.`MAIN` as sink WHERE (sink.`BATCH_ID_OUT` = 999999999) AND " + - "(((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-01')) " + - "OR ((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-02')) " + - "OR ((sink.`ACCOUNT_TYPE` = 'TYPE_2') AND (sink.`BIZ_DATE` = '2024-01-02'))))))"; + "(((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-01')) " + + "OR ((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-02')) " + + "OR ((sink.`ACCOUNT_TYPE` = 2) AND (sink.`BIZ_DATE` = '2024-01-02'))))))"; Assertions.assertEquals(MemsqlTestArtifacts.expectedMainTableMultiPartitionsCreateQueryWithUpperCase, preActionsSql.get(0)); Assertions.assertEquals(MemsqlTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); @@ -259,9 +259,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListWithEmptyBatchHandling String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + "sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + "WHERE (sink.`batch_id_out` = 999999999) AND " + - "(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) " + - "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2')))"; + "(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) " + + "OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2)))"; Assertions.assertEquals(MemsqlTestArtifacts.expectedMainTableMultiPartitionKeysCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(MemsqlTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-postgres/src/test/java/org/finos/legend/engine/persistence/components/PostgresTestArtifacts.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-postgres/src/test/java/org/finos/legend/engine/persistence/components/PostgresTestArtifacts.java index 5ccf0e9c411..64b88928ef1 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-postgres/src/test/java/org/finos/legend/engine/persistence/components/PostgresTestArtifacts.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-postgres/src/test/java/org/finos/legend/engine/persistence/components/PostgresTestArtifacts.java @@ -64,11 +64,11 @@ public class PostgresTestArtifacts "\"batch_id_in\" INTEGER NOT NULL,\"batch_id_out\" INTEGER,PRIMARY KEY (\"id\", \"name\", \"batch_id_in\"))"; public static String expectedMainTableMultiPartitionsCreateQuery = "CREATE TABLE IF NOT EXISTS \"mydb\".\"main\"(" + - "\"id\" INTEGER NOT NULL,\"name\" VARCHAR NOT NULL,\"amount\" DOUBLE PRECISION,\"account_type\" VARCHAR,\"biz_date\" DATE,\"digest\" VARCHAR," + + "\"id\" INTEGER NOT NULL,\"name\" VARCHAR NOT NULL,\"amount\" DOUBLE PRECISION,\"account_type\" INTEGER,\"biz_date\" DATE,\"digest\" VARCHAR," + "\"batch_id_in\" INTEGER NOT NULL,\"batch_id_out\" INTEGER,PRIMARY KEY (\"id\", \"name\", \"batch_id_in\"))"; public static String expectedMainTableMultiPartitionsCreateQueryUpperCase = "CREATE TABLE IF NOT EXISTS \"MYDB\".\"MAIN\"" + - "(\"ID\" INTEGER NOT NULL,\"NAME\" VARCHAR NOT NULL,\"AMOUNT\" DOUBLE PRECISION,\"ACCOUNT_TYPE\" VARCHAR,\"BIZ_DATE\" DATE," + + "(\"ID\" INTEGER NOT NULL,\"NAME\" VARCHAR NOT NULL,\"AMOUNT\" DOUBLE PRECISION,\"ACCOUNT_TYPE\" INTEGER,\"BIZ_DATE\" DATE," + "\"DIGEST\" VARCHAR,\"BATCH_ID_IN\" INTEGER NOT NULL,\"BATCH_ID_OUT\" INTEGER,PRIMARY KEY (\"ID\", \"NAME\", \"BATCH_ID_IN\"))"; public static String expectedMainTableBatchIdAndVersionBasedCreateQuery = "CREATE TABLE IF NOT EXISTS \"mydb\".\"main\"(" + diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-postgres/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotBatchIdBasedTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-postgres/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotBatchIdBasedTest.java index 627cbdfba12..fec061feda5 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-postgres/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotBatchIdBasedTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-postgres/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotBatchIdBasedTest.java @@ -219,9 +219,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener "\"batch_id_out\" = (SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')-1 " + "WHERE (sink.\"batch_id_out\" = 999999999) AND " + "(NOT (EXISTS (SELECT * FROM \"mydb\".\"staging\" as stage WHERE ((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (sink.\"digest\" = stage.\"digest\")))) " + - "AND (((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2')))"; + "AND (((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2)))"; String expectedUpsertQuery = "INSERT INTO \"mydb\".\"main\" " + "(\"id\", \"name\", \"amount\", \"account_type\", \"biz_date\", \"digest\", \"batch_id_in\", \"batch_id_out\") " + @@ -229,9 +229,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')," + "999999999 FROM \"mydb\".\"staging\" as stage WHERE " + "NOT (stage.\"digest\" IN (SELECT sink.\"digest\" FROM \"mydb\".\"main\" as sink WHERE (sink.\"batch_id_out\" = 999999999) " + - "AND (((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2'))))))"; + "AND (((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2))))))"; Assertions.assertEquals(PostgresTestArtifacts.expectedMainTableMultiPartitionsCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(getExpectedMetadataTableCreateQuery(), preActionsSql.get(1)); @@ -253,9 +253,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe "WHERE (sink.\"BATCH_ID_OUT\" = 999999999) AND " + "(NOT (EXISTS (SELECT * FROM \"MYDB\".\"STAGING\" as stage " + "WHERE ((sink.\"ID\" = stage.\"ID\") AND (sink.\"NAME\" = stage.\"NAME\")) AND (sink.\"DIGEST\" = stage.\"DIGEST\")))) AND " + - "(((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-01')) " + - "OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-02')) " + - "OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_2') AND (sink.\"BIZ_DATE\" = '2024-01-02')))"; + "(((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-01')) " + + "OR ((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-02')) " + + "OR ((sink.\"ACCOUNT_TYPE\" = 2) AND (sink.\"BIZ_DATE\" = '2024-01-02')))"; String expectedUpsertQuery = "INSERT INTO \"MYDB\".\"MAIN\" " + "(\"ID\", \"NAME\", \"AMOUNT\", \"ACCOUNT_TYPE\", \"BIZ_DATE\", \"DIGEST\", \"BATCH_ID_IN\", \"BATCH_ID_OUT\") " + @@ -263,9 +263,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe "(SELECT COALESCE(MAX(BATCH_METADATA.\"TABLE_BATCH_ID\"),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.\"TABLE_NAME\") = 'MAIN')," + "999999999 FROM \"MYDB\".\"STAGING\" as stage " + "WHERE NOT (stage.\"DIGEST\" IN (SELECT sink.\"DIGEST\" FROM \"MYDB\".\"MAIN\" as sink WHERE (sink.\"BATCH_ID_OUT\" = 999999999) " + - "AND (((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-01')) " + - "OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-02')) " + - "OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_2') AND (sink.\"BIZ_DATE\" = '2024-01-02'))))))"; + "AND (((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-01')) " + + "OR ((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-02')) " + + "OR ((sink.\"ACCOUNT_TYPE\" = 2) AND (sink.\"BIZ_DATE\" = '2024-01-02'))))))"; Assertions.assertEquals(PostgresTestArtifacts.expectedMainTableMultiPartitionsCreateQueryUpperCase, preActionsSql.get(0)); Assertions.assertEquals(getExpectedMetadataTableCreateQueryWithUpperCase(), preActionsSql.get(1)); @@ -284,9 +284,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListWithEmptyBatchHandling String expectedMilestoneQuery = "UPDATE \"mydb\".\"main\" as sink " + "SET \"batch_id_out\" = (SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')-1 " + "WHERE (sink.\"batch_id_out\" = 999999999) AND " + - "(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " + - "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2')))"; + "(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " + + "OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2)))"; Assertions.assertEquals(PostgresTestArtifacts.expectedMainTableMultiPartitionsCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(getExpectedMetadataTableCreateQuery(), preActionsSql.get(1)); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/src/test/java/org/finos/legend/engine/persistence/components/BaseTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/src/test/java/org/finos/legend/engine/persistence/components/BaseTest.java index 777779426fb..a949b7d73fa 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/src/test/java/org/finos/legend/engine/persistence/components/BaseTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/src/test/java/org/finos/legend/engine/persistence/components/BaseTest.java @@ -114,25 +114,25 @@ public class BaseTest protected Map> partitionFilterWithMultiValuesForMultipleKeys = new HashMap>() {{ put("biz_date", new HashSet<>(Arrays.asList("2000-01-01 00:00:00", "2000-01-02 00:00:00"))); - put("account_type", new HashSet<>(Arrays.asList("TYPE_1", "TYPE_2"))); + put("account_type", new HashSet<>(Arrays.asList("1", "2"))); }}; protected Map> partitionFilterWithMultiValuesForOneKey = new HashMap>() {{ put("biz_date", new HashSet<>(Arrays.asList("2000-01-01 00:00:00", "2000-01-02 00:00:00"))); - put("account_type", new HashSet<>(Arrays.asList("TYPE_1"))); + put("account_type", new HashSet<>(Arrays.asList("1"))); }}; protected List> partitionSpecList() { List> partitionSpecList = new ArrayList<>(); - addPartitionSpec(partitionSpecList, "2024-01-01", "TYPE_1"); - addPartitionSpec(partitionSpecList, "2024-01-02", "TYPE_1"); - addPartitionSpec(partitionSpecList, "2024-01-02", "TYPE_2"); + addPartitionSpec(partitionSpecList, "2024-01-01", 1); + addPartitionSpec(partitionSpecList, "2024-01-02", 1); + addPartitionSpec(partitionSpecList, "2024-01-02", 2); return partitionSpecList; } - private static void addPartitionSpec(List> partitionSpecList, String date, String accountType) + private static void addPartitionSpec(List> partitionSpecList, String date, Integer accountType) { partitionSpecList.add(new HashMap() { @@ -150,7 +150,7 @@ private static void addPartitionSpec(List> partitionSpecList protected Field name = Field.builder().name("name").type(FieldType.of(DataType.VARCHAR, Optional.empty(), Optional.empty())).primaryKey(true).build(); protected Field nameModified = Field.builder().name("name").type(FieldType.of(DataType.VARCHAR, 64, null)).primaryKey(true).build(); protected Field amount = Field.builder().name("amount").type(FieldType.of(DataType.DOUBLE, Optional.empty(), Optional.empty())).build(); - protected Field accountType = Field.builder().name("account_type").type(FieldType.of(DataType.VARCHAR, Optional.empty(), Optional.empty())).build(); + protected Field accountType = Field.builder().name("account_type").type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())).build(); protected Field floatAmount = Field.builder().name("amount").type(FieldType.of(DataType.FLOAT, Optional.empty(), Optional.empty())).build(); protected Field bizDate = Field.builder().name("biz_date").type(FieldType.of(DataType.DATE, Optional.empty(), Optional.empty())).build();