Skip to content

Commit

Permalink
Fix the tests
Browse files Browse the repository at this point in the history
  • Loading branch information
prasar-ashutosh committed May 3, 2024
1 parent 93cda15 commit f78869f
Show file tree
Hide file tree
Showing 10 changed files with 74 additions and 74 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ public static Condition getPartitionSpecMatchCondition(Dataset dataSet, List<Map
.map(columnValuePair ->
Equals.of(
FieldValue.builder().datasetRef(dataSet.datasetReference()).fieldName(columnValuePair.getKey()).build(),
StringValue.of((String) columnValuePair.getValue()))
columnValuePair.getValue() instanceof Number ? ObjectValue.of(columnValuePair.getValue()) : StringValue.of((String) columnValuePair.getValue()))
)
.collect(Collectors.toList()))
).collect(Collectors.toList()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public class AnsiTestArtifacts
"\"batch_id_in\" INTEGER NOT NULL,\"batch_id_out\" INTEGER,PRIMARY KEY (\"id\", \"name\", \"batch_id_in\"))";

public static String expectedMainTableWithMultiPartitionsCreateQuery = "CREATE TABLE IF NOT EXISTS \"mydb\".\"main\"(" +
"\"id\" INTEGER NOT NULL,\"name\" VARCHAR NOT NULL,\"amount\" DOUBLE,\"account_type\" VARCHAR,\"biz_date\" DATE,\"digest\" VARCHAR," +
"\"id\" INTEGER NOT NULL,\"name\" VARCHAR NOT NULL,\"amount\" DOUBLE,\"account_type\" INTEGER,\"biz_date\" DATE,\"digest\" VARCHAR," +
"\"batch_id_in\" INTEGER NOT NULL,\"batch_id_out\" INTEGER,PRIMARY KEY (\"id\", \"name\", \"batch_id_in\"))";

public static String expectedMainTableBatchIdAndVersionBasedCreateQuery = "CREATE TABLE IF NOT EXISTS \"mydb\".\"main\"(" +
Expand All @@ -80,7 +80,7 @@ public class AnsiTestArtifacts
"\"BATCH_ID_IN\" INTEGER NOT NULL,\"BATCH_ID_OUT\" INTEGER,PRIMARY KEY (\"ID\", \"NAME\", \"BATCH_ID_IN\"))";

public static String expectedMainTableMultiPartitionCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS \"MYDB\".\"MAIN\"" +
"(\"ID\" INTEGER NOT NULL,\"NAME\" VARCHAR NOT NULL,\"AMOUNT\" DOUBLE,\"ACCOUNT_TYPE\" VARCHAR,\"BIZ_DATE\" DATE,\"DIGEST\" VARCHAR," +
"(\"ID\" INTEGER NOT NULL,\"NAME\" VARCHAR NOT NULL,\"AMOUNT\" DOUBLE,\"ACCOUNT_TYPE\" INTEGER,\"BIZ_DATE\" DATE,\"DIGEST\" VARCHAR," +
"\"BATCH_ID_IN\" INTEGER NOT NULL,\"BATCH_ID_OUT\" INTEGER,PRIMARY KEY (\"ID\", \"NAME\", \"BATCH_ID_IN\"))";

public static String expectedMainTableTimeBasedCreateQuery = "CREATE TABLE IF NOT EXISTS \"mydb\".\"main\"(" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,9 +219,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener
"WHERE (sink.\"batch_id_out\" = 999999999) AND " +
"(NOT (EXISTS (SELECT * FROM \"mydb\".\"staging\" as stage " +
"WHERE ((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (sink.\"digest\" = stage.\"digest\")))) " +
"AND (((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2')))";
"AND (((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2)))";

String expectedUpsertQuery = "INSERT INTO \"mydb\".\"main\" " +
"(\"id\", \"name\", \"amount\", \"account_type\", \"biz_date\", \"digest\", \"batch_id_in\", \"batch_id_out\") " +
Expand All @@ -230,9 +230,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener
"FROM \"mydb\".\"staging\" as stage WHERE " +
"NOT (stage.\"digest\" IN (SELECT sink.\"digest\" FROM \"mydb\".\"main\" as sink WHERE " +
"(sink.\"batch_id_out\" = 999999999) AND " +
"(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2'))))))";
"(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2))))))";

Assertions.assertEquals(AnsiTestArtifacts.expectedMainTableWithMultiPartitionsCreateQuery, preActionsSql.get(0));
Assertions.assertEquals(getExpectedMetadataTableCreateQuery(), preActionsSql.get(1));
Expand All @@ -254,19 +254,19 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe
"WHERE (sink.\"BATCH_ID_OUT\" = 999999999) AND " +
"(NOT (EXISTS (SELECT * FROM \"MYDB\".\"STAGING\" as stage " +
"WHERE ((sink.\"ID\" = stage.\"ID\") AND (sink.\"NAME\" = stage.\"NAME\")) AND (sink.\"DIGEST\" = stage.\"DIGEST\")))) " +
"AND (((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-01')) " +
"OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-02')) " +
"OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_2') AND (sink.\"BIZ_DATE\" = '2024-01-02')))";
"AND (((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-01')) " +
"OR ((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-02')) " +
"OR ((sink.\"ACCOUNT_TYPE\" = 2) AND (sink.\"BIZ_DATE\" = '2024-01-02')))";

String expectedUpsertQuery = "INSERT INTO \"MYDB\".\"MAIN\" " +
"(\"ID\", \"NAME\", \"AMOUNT\", \"ACCOUNT_TYPE\", \"BIZ_DATE\", \"DIGEST\", \"BATCH_ID_IN\", \"BATCH_ID_OUT\") " +
"(SELECT stage.\"ID\",stage.\"NAME\",stage.\"AMOUNT\",stage.\"ACCOUNT_TYPE\",stage.\"BIZ_DATE\",stage.\"DIGEST\"," +
"(SELECT COALESCE(MAX(BATCH_METADATA.\"TABLE_BATCH_ID\"),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.\"TABLE_NAME\") = 'MAIN')," +
"999999999 FROM \"MYDB\".\"STAGING\" as stage " +
"WHERE NOT (stage.\"DIGEST\" IN (SELECT sink.\"DIGEST\" FROM \"MYDB\".\"MAIN\" as sink WHERE (sink.\"BATCH_ID_OUT\" = 999999999) " +
"AND (((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-01')) " +
"OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_1') AND (sink.\"BIZ_DATE\" = '2024-01-02')) " +
"OR ((sink.\"ACCOUNT_TYPE\" = 'TYPE_2') AND (sink.\"BIZ_DATE\" = '2024-01-02'))))))";
"AND (((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-01')) " +
"OR ((sink.\"ACCOUNT_TYPE\" = 1) AND (sink.\"BIZ_DATE\" = '2024-01-02')) " +
"OR ((sink.\"ACCOUNT_TYPE\" = 2) AND (sink.\"BIZ_DATE\" = '2024-01-02'))))))";

Assertions.assertEquals(AnsiTestArtifacts.expectedMainTableMultiPartitionCreateQueryWithUpperCase, preActionsSql.get(0));
Assertions.assertEquals(getExpectedMetadataTableCreateQueryWithUpperCase(), preActionsSql.get(1));
Expand All @@ -285,9 +285,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListWithEmptyBatchHandling
String expectedMilestoneQuery = "UPDATE \"mydb\".\"main\" as sink SET " +
"sink.\"batch_id_out\" = (SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')-1 " +
"WHERE (sink.\"batch_id_out\" = 999999999) AND " +
"(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 'TYPE_1')) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_1')) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 'TYPE_2')))";
"(((sink.\"biz_date\" = '2024-01-01') AND (sink.\"account_type\" = 1)) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 1)) " +
"OR ((sink.\"biz_date\" = '2024-01-02') AND (sink.\"account_type\" = 2)))";

Assertions.assertEquals(AnsiTestArtifacts.expectedMainTableWithMultiPartitionsCreateQuery, preActionsSql.get(0));
Assertions.assertEquals(getExpectedMetadataTableCreateQuery(), preActionsSql.get(1));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,11 +202,11 @@ public class BigQueryTestArtifacts
"`batch_id_in` INT64 NOT NULL,`batch_id_out` INT64,PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)";

public static String expectedMainTableWithMultiPartitionCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" +
"`id` INT64 NOT NULL,`name` STRING NOT NULL,`amount` FLOAT64,`account_type` STRING,`biz_date` DATE,`digest` STRING," +
"`id` INT64 NOT NULL,`name` STRING NOT NULL,`amount` FLOAT64,`account_type` INT64,`biz_date` DATE,`digest` STRING," +
"`batch_id_in` INT64 NOT NULL,`batch_id_out` INT64,PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)";

public static String expectedMainTableWithMultiPartitionCreateQueryUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" +
"(`ID` INT64 NOT NULL,`NAME` STRING NOT NULL,`AMOUNT` FLOAT64,`ACCOUNT_TYPE` STRING,`BIZ_DATE` DATE,`DIGEST` STRING," +
"(`ID` INT64 NOT NULL,`NAME` STRING NOT NULL,`AMOUNT` FLOAT64,`ACCOUNT_TYPE` INT64,`BIZ_DATE` DATE,`DIGEST` STRING," +
"`BATCH_ID_IN` INT64 NOT NULL,`BATCH_ID_OUT` INT64,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`) NOT ENFORCED)";

public static String expectedMetadataTableCreateQuery = "CREATE TABLE IF NOT EXISTS batch_metadata" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,9 +194,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener
"WHERE (sink.`batch_id_out` = 999999999) AND " +
"(NOT (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE " +
"((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) AND " +
"(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) " +
"OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) " +
"OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2')))";
"(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) " +
"OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) " +
"OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2)))";

String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " +
"(`id`, `name`, `amount`, `account_type`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " +
Expand All @@ -205,9 +205,9 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersion(Gener
"WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " +
"FROM `mydb`.`staging` as stage WHERE " +
"NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND " +
"(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) " +
"OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) " +
"OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2'))))))";
"(((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) " +
"OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) " +
"OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2))))))";

Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableWithMultiPartitionCreateQuery, preActionsSql.get(0));
Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1));
Expand All @@ -228,19 +228,19 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListNoDedupNoVersionInUppe
"sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1 " +
"WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (NOT (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage " +
"WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) AND (sink.`DIGEST` = stage.`DIGEST`)))) " +
"AND (((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-01')) " +
"OR ((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-02')) " +
"OR ((sink.`ACCOUNT_TYPE` = 'TYPE_2') AND (sink.`BIZ_DATE` = '2024-01-02')))";
"AND (((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-01')) " +
"OR ((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-02')) " +
"OR ((sink.`ACCOUNT_TYPE` = 2) AND (sink.`BIZ_DATE` = '2024-01-02')))";

String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` " +
"(`ID`, `NAME`, `AMOUNT`, `ACCOUNT_TYPE`, `BIZ_DATE`, `DIGEST`, `BATCH_ID_IN`, `BATCH_ID_OUT`) " +
"(SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`ACCOUNT_TYPE`,stage.`BIZ_DATE`,stage.`DIGEST`," +
"(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')," +
"999999999 FROM `MYDB`.`STAGING` as stage WHERE " +
"NOT (stage.`DIGEST` IN (SELECT sink.`DIGEST` FROM `MYDB`.`MAIN` as sink WHERE (sink.`BATCH_ID_OUT` = 999999999) AND " +
"(((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-01')) " +
"OR ((sink.`ACCOUNT_TYPE` = 'TYPE_1') AND (sink.`BIZ_DATE` = '2024-01-02')) " +
"OR ((sink.`ACCOUNT_TYPE` = 'TYPE_2') AND (sink.`BIZ_DATE` = '2024-01-02'))))))";
"(((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-01')) " +
"OR ((sink.`ACCOUNT_TYPE` = 1) AND (sink.`BIZ_DATE` = '2024-01-02')) " +
"OR ((sink.`ACCOUNT_TYPE` = 2) AND (sink.`BIZ_DATE` = '2024-01-02'))))))";

Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableWithMultiPartitionCreateQueryUpperCase, preActionsSql.get(0));
Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1));
Expand All @@ -256,7 +256,7 @@ public void verifyUnitemporalSnapshotWithPartitionSpecListWithEmptyBatchHandling
List<String> milestoningSql = operations.ingestSql();
List<String> metadataIngestSql = operations.metadataIngestSql();

String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 WHERE (sink.`batch_id_out` = 999999999) AND (((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 'TYPE_1')) OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_1')) OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 'TYPE_2')))";
String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 WHERE (sink.`batch_id_out` = 999999999) AND (((sink.`biz_date` = '2024-01-01') AND (sink.`account_type` = 1)) OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 1)) OR ((sink.`biz_date` = '2024-01-02') AND (sink.`account_type` = 2)))";

Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableWithMultiPartitionCreateQuery, preActionsSql.get(0));
Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ public class MemsqlTestArtifacts
"`batch_id_in` INTEGER NOT NULL,`batch_id_out` INTEGER,PRIMARY KEY (`id`, `name`, `batch_id_in`))";

public static String expectedMainTableMultiPartitionKeysCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" +
"`id` INTEGER NOT NULL,`name` VARCHAR(256) NOT NULL,`amount` DOUBLE,`account_type` VARCHAR(256),`biz_date` DATE,`digest` VARCHAR(256)," +
"`id` INTEGER NOT NULL,`name` VARCHAR(256) NOT NULL,`amount` DOUBLE,`account_type` INTEGER,`biz_date` DATE,`digest` VARCHAR(256)," +
"`batch_id_in` INTEGER NOT NULL,`batch_id_out` INTEGER,PRIMARY KEY (`id`, `name`, `batch_id_in`))";

public static String expectedMetadataTableCreateQuery = "CREATE TABLE IF NOT EXISTS batch_metadata" +
Expand All @@ -219,7 +219,7 @@ public class MemsqlTestArtifacts
"`BATCH_ID_IN` INTEGER NOT NULL,`BATCH_ID_OUT` INTEGER,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`))";

public static String expectedMainTableMultiPartitionsCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" +
"(`ID` INTEGER NOT NULL,`NAME` VARCHAR(256) NOT NULL,`AMOUNT` DOUBLE,`ACCOUNT_TYPE` VARCHAR(256),`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," +
"(`ID` INTEGER NOT NULL,`NAME` VARCHAR(256) NOT NULL,`AMOUNT` DOUBLE,`ACCOUNT_TYPE` INTEGER,`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," +
"`BATCH_ID_IN` INTEGER NOT NULL,`BATCH_ID_OUT` INTEGER,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`))";

public static String expectedMetadataTableIngestQuery = "INSERT INTO batch_metadata (`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`)" +
Expand Down
Loading

0 comments on commit f78869f

Please sign in to comment.