Skip to content

Commit

Permalink
Merge pull request #1453 from cloudsufi/Added-cleanuphooks-e2e
Browse files Browse the repository at this point in the history 
Added cleanup code for gcp resources
  • Loading branch information
@itsankit-google
itsankit-google authored Oct 23, 2024
2 parents 283824a + 4523759 commit b242c36
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 14 deletions.
2 changes: 1 addition & 1 deletion src/e2e-test/features/bigquery/sink/BigQueryToBigQueryAdditional.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
Then Verify the pipeline status is "Succeeded"
Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqUpsertDedupeFile"

@BQ_RECORD_SOURCE_TEST @BQ_SECOND_RECORD_SOURCE_TEST @BQ_SINK_TEST
@BQ_PRIMARY_RECORD_SOURCE_TEST @BQ_SECONDARY_RECORD_SOURCE_TEST @BQ_SINK_TEST
Scenario: Validate successful record transfer from two BigQuery source plugins with different schema record names, taking one extra column in BigQuery source plugin 1,and
using wrangler transformation plugin for removing the extra column and transferring the data in BigQuery sink plugin containing all the columns from both the source plugin.
Given Open Datafusion Project to configure pipeline
Expand Down
36 changes: 23 additions & 13 deletions src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,8 @@ public static void createBucketWithAvroTestFile() throws IOException, URISyntaxE
@After(order = 1, value = "@GCS_CSV_TEST or @GCS_TSV_TEST or @GCS_BLOB_TEST " +
"or @GCS_DELIMITED_TEST or @GCS_TEXT_TEST or @GCS_OUTPUT_FIELD_TEST or @GCS_DATATYPE_1_TEST or " +
"@GCS_DATATYPE_2_TEST or @GCS_READ_RECURSIVE_TEST or @GCS_DELETE_WILDCARD_TEST or @GCS_CSV_RANGE_TEST or" +
" @GCS_PARQUET_TEST or @GCS_AVRO_TEST or @GCS_DATATYPE_TEST or @GCS_AVRO_FILE")
" @GCS_PARQUET_TEST or @GCS_AVRO_TEST or @GCS_DATATYPE_TEST or @GCS_AVRO_FILE or @GCS_CSV or " +
"GCS_MULTIPLE_FILES_TEST or GCS_MULTIPLE_FILES_REGEX_TEST")
public static void deleteSourceBucketWithFile() {
deleteGCSBucket(gcsSourceBucketName);
PluginPropertyUtils.removePluginProp("gcsSourceBucketName");
Expand Down Expand Up @@ -235,7 +236,7 @@ public static void createTargetGCSBucketWithCSVFile() throws IOException, URISyn
BeforeActions.scenario.write("GCS target bucket name - " + gcsTargetBucketName);
}

@After(order = 1, value = "@GCS_SINK_TEST or @GCS_SINK_EXISTING_BUCKET_TEST")
@After(order = 1, value = "@GCS_SINK_TEST or @GCS_SINK_EXISTING_BUCKET_TEST or @GCS_SINK_MULTI_PART_UPLOAD")
public static void deleteTargetBucketWithFile() {
deleteGCSBucket(gcsTargetBucketName);
PluginPropertyUtils.removePluginProp("gcsTargetBucketName");
Expand All @@ -250,7 +251,8 @@ public static void setTempTargetBQTableName() {
BeforeActions.scenario.write("BQ Target table name - " + bqTargetTable);
}

@After(order = 1, value = "@BQ_SINK_TEST")
@After(order = 1, value = "@BQ_SINK_TEST or @BQ_UPSERT_SINK_TEST or @BQ_UPDATE_SINK_DEDUPE_TEST or " +
"@BQ_EXISTING_SINK_TEST or @BQ_UPSERT_DEDUPE_SINK_TEST or @BQ_INSERT_SINK_TEST")
public static void deleteTempTargetBQTable() throws IOException, InterruptedException {
try {
BigQueryClient.dropBqQuery(bqTargetTable);
Expand Down Expand Up @@ -294,7 +296,9 @@ public static void createTempSourceBQTable() throws IOException, InterruptedExce

@After(order = 1, value = "@BQ_SOURCE_TEST or @BQ_PARTITIONED_SOURCE_TEST or @BQ_SOURCE_DATATYPE_TEST or " +
"@BQ_INSERT_SOURCE_TEST or @BQ_UPDATE_SINK_TEST or @BQ_EXISTING_SOURCE_TEST or @BQ_EXISTING_SINK_TEST or " +
"@BQ_EXISTING_SOURCE_DATATYPE_TEST or @BQ_EXISTING_SINK_DATATYPE_TEST")
"@BQ_EXISTING_SOURCE_DATATYPE_TEST or @BQ_EXISTING_SINK_DATATYPE_TEST or @BQ_UPSERT_SOURCE_TEST or " +
"@BQ_NULL_MODE_SOURCE_TEST or @BQ_UPDATE_SOURCE_DEDUPE_TEST or @BQ_INSERT_INT_SOURCE_TEST or " +
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST")
public static void deleteTempSourceBQTable() throws IOException, InterruptedException {
BigQueryClient.dropBqQuery(bqSourceTable);
PluginPropertyUtils.removePluginProp("bqSourceTable");
Expand Down Expand Up @@ -1251,7 +1255,7 @@ public static void createSinkBQDeupeUpsertTable() throws IOException, Interrupte
BeforeActions.scenario.write("BQ Target Table " + bqTargetTable + " created successfully");
}

@Before(order = 1, value = "@BQ_RECORD_SOURCE_TEST")
@Before(order = 1, value = "@BQ_PRIMARY_RECORD_SOURCE_TEST")
public static void createSourceBQRecordTable() throws IOException, InterruptedException {
bqSourceTable = "E2E_SOURCE_" + UUID.randomUUID().toString().replaceAll("-", "_");
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqSourceTable + "` " +
Expand All @@ -1270,7 +1274,7 @@ public static void createSourceBQRecordTable() throws IOException, InterruptedEx
BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " created successfully");
}

@Before(order = 1, value = "@BQ_SECOND_RECORD_SOURCE_TEST")
@Before(order = 1, value = "@BQ_SECONDARY_RECORD_SOURCE_TEST")
public static void createSourceBQSecondRecordTable() throws IOException, InterruptedException {
bqSourceTable2 = "E2E_SOURCE_" + UUID.randomUUID().toString().replaceAll("-", "_");
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqSourceTable2 + "` " +
Expand Down Expand Up @@ -1604,18 +1608,16 @@ public static void createSourceTables() throws IOException, InterruptedException
public static void deleteAllBqTables() throws IOException, InterruptedException {
BigQueryClient.dropBqQuery(bqSourceTable);
BigQueryClient.dropBqQuery(bqSourceTable2);
BigQueryClient.dropBqQuery(PluginPropertyUtils.pluginProp("bqTargetTable"));
BigQueryClient.dropBqQuery(PluginPropertyUtils.pluginProp("bqTargetTable2"));
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
BigQueryClient.dropBqQuery(bqTargetTable);
BigQueryClient.dropBqQuery(bqTargetTable2);
PluginPropertyUtils.removePluginProp("bqSourceTable");
PluginPropertyUtils.removePluginProp("bqSourceTable2");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
BeforeActions.scenario.write("BQ target Table " + bqTargetTable + " deleted successfully");
BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable + " deleted successfully");
bqSourceTable = StringUtils.EMPTY;
bqSourceTable2 = StringUtils.EMPTY;
bqTargetTable = StringUtils.EMPTY;
bqTargetTable2 = StringUtils.EMPTY;
BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable2 + " deleted successfully");
}

@After(order = 1, value = "@BQ_SINK_BQMT_TEST")
Expand All @@ -1636,4 +1638,12 @@ public static void deleteTargetBqmtTable() throws IOException, InterruptedExcept
}
}
}

@After(order = 1, value = "@BQ_SECONDARY_RECORD_SOURCE_TEST")
public static void deleteTempSource2BQTable() throws IOException, InterruptedException {
BigQueryClient.dropBqQuery(bqSourceTable2);
bqSourceTable2 = PluginPropertyUtils.pluginProp("bqSourceTable2");
PluginPropertyUtils.removePluginProp("bqSourceTable2");
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
}
}

0 comments on commit b242c36

Please sign in to comment.