From 90cb8f90216b320341ec5e6b600322d7f69018f1 Mon Sep 17 00:00:00 2001 From: Sean Kao Date: Fri, 4 Oct 2024 16:29:31 -0700 Subject: [PATCH] test suites unset custom config when test failure Signed-off-by: Sean Kao --- .../flint/spark/FlintSparkIndexBuilderSuite.scala | 14 +++++++------- .../spark/FlintSparkCoveringIndexITSuite.scala | 8 ++------ .../spark/FlintSparkCoveringIndexSqlITSuite.scala | 5 ++--- .../spark/FlintSparkMaterializedViewITSuite.scala | 8 ++------ .../FlintSparkMaterializedViewSqlITSuite.scala | 4 ++-- .../spark/FlintSparkSkippingIndexITSuite.scala | 7 ++----- .../spark/FlintSparkSkippingIndexSqlITSuite.scala | 5 ++--- 7 files changed, 19 insertions(+), 32 deletions(-) diff --git a/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexBuilderSuite.scala b/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexBuilderSuite.scala index a2ec85df9..35bd569c5 100644 --- a/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexBuilderSuite.scala +++ b/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexBuilderSuite.scala @@ -38,6 +38,13 @@ class FlintSparkIndexBuilderSuite extends FlintSuite { super.afterAll() } + override def afterEach(): Unit = { + super.afterEach() + conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) + conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_INTERVAL_THRESHOLD.key) + conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) + } + test("indexOptions should not have checkpoint location when no conf") { assert(!conf.contains(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key)) @@ -210,13 +217,6 @@ class FlintSparkIndexBuilderSuite extends FlintSuite { "External scheduler mode is not enabled in the configuration") } - override def afterEach(): Unit = { - super.afterEach() - conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) - conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_INTERVAL_THRESHOLD.key) - conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) - } - private def builder(): FakeFlintSparkIndexBuilder = { new FakeFlintSparkIndexBuilder } diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala index 54d3ba6dc..ca2eeed1e 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala @@ -39,7 +39,8 @@ class FlintSparkCoveringIndexITSuite extends FlintSparkSuite { override def afterEach(): Unit = { super.afterEach() - + conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) + conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) // Delete all test indices deleteTestIndex(testFlintIndex) } @@ -162,8 +163,6 @@ class FlintSparkCoveringIndexITSuite extends FlintSparkSuite { assert( checkpointLocation.get.contains(testFlintIndex), s"Checkpoint location dir should contain ${testFlintIndex}") - - conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) } } @@ -209,7 +208,6 @@ class FlintSparkCoveringIndexITSuite extends FlintSparkSuite { val indexData = flint.queryIndex(testFlintIndex) checkAnswer(indexData, Seq(Row("Hello", 30), Row("World", 25))) - conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) } } @@ -289,8 +287,6 @@ class FlintSparkCoveringIndexITSuite extends FlintSparkSuite { assert( checkpointLocation.get.contains(testFlintIndex), s"Checkpoint location dir should contain ${testFlintIndex}") - - conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) } } diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexSqlITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexSqlITSuite.scala index aac06a2c1..415e5027f 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexSqlITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexSqlITSuite.scala @@ -37,7 +37,8 @@ class FlintSparkCoveringIndexSqlITSuite extends FlintSparkSuite { override def afterEach(): Unit = { super.afterEach() - + conf.unsetConf(FlintSparkConf.CUSTOM_FLINT_SCHEDULER_CLASS.key) + conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) // Delete all test indices deleteTestIndex(testFlintIndex) sql(s"DROP TABLE $testTable") @@ -180,8 +181,6 @@ class FlintSparkCoveringIndexSqlITSuite extends FlintSparkSuite { // Drop index with test scheduler sql(s"DROP INDEX $testIndex ON $testTable") - conf.unsetConf(FlintSparkConf.CUSTOM_FLINT_SCHEDULER_CLASS.key) - conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) } } diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewITSuite.scala index c00e982e0..17959d320 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewITSuite.scala @@ -48,6 +48,8 @@ class FlintSparkMaterializedViewITSuite extends FlintSparkSuite { override def afterEach(): Unit = { super.afterEach() + conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) + conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) deleteTestIndex(testFlintIndex) } @@ -130,8 +132,6 @@ class FlintSparkMaterializedViewITSuite extends FlintSparkSuite { assert( checkpointLocation.get.contains(testFlintIndex), s"Checkpoint location dir should contain ${testFlintIndex}") - - conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) } } @@ -333,8 +333,6 @@ class FlintSparkMaterializedViewITSuite extends FlintSparkSuite { assert( checkpointLocation.get.contains(testFlintIndex), s"Checkpoint location dir should contain ${testFlintIndex}") - - conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) } } @@ -385,8 +383,6 @@ class FlintSparkMaterializedViewITSuite extends FlintSparkSuite { Row(timestamp("2023-10-01 00:00:00"), 1), Row(timestamp("2023-10-01 00:10:00"), 2), Row(timestamp("2023-10-01 01:00:00"), 1))) - - conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) } } diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala index 688e2e72d..e12c375cf 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala @@ -45,6 +45,8 @@ class FlintSparkMaterializedViewSqlITSuite extends FlintSparkSuite { override def afterEach(): Unit = { super.afterEach() + conf.unsetConf(FlintSparkConf.CUSTOM_FLINT_SCHEDULER_CLASS.key) + conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) deleteTestIndex(testFlintIndex) sql(s"DROP TABLE $testTable") } @@ -119,8 +121,6 @@ class FlintSparkMaterializedViewSqlITSuite extends FlintSparkSuite { // Drop index with test scheduler sql(s"DROP MATERIALIZED VIEW $testMvName") - conf.unsetConf(FlintSparkConf.CUSTOM_FLINT_SCHEDULER_CLASS.key) - conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) } } diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala index b535173e3..dc1a472c7 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala @@ -49,6 +49,8 @@ class FlintSparkSkippingIndexITSuite extends FlintSparkSuite { } override def afterEach(): Unit = { + conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) + conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) // Delete all test indices deleteTestIndex(testIndex) sql(s"DROP TABLE $testTable") @@ -215,8 +217,6 @@ class FlintSparkSkippingIndexITSuite extends FlintSparkSuite { assert( checkpointLocation.get.contains(testIndex), s"Checkpoint location dir should contain ${testIndex}") - - conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) } } @@ -363,7 +363,6 @@ class FlintSparkSkippingIndexITSuite extends FlintSparkSuite { // Expect to only refresh the new file flint.refreshIndex(testIndex) shouldBe empty flint.queryIndex(testIndex).collect().toSet should have size 1 - conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) } } @@ -442,8 +441,6 @@ class FlintSparkSkippingIndexITSuite extends FlintSparkSuite { assert( checkpointLocation.get.contains(testIndex), s"Checkpoint location dir should contain ${testIndex}") - - conf.unsetConf(FlintSparkConf.CHECKPOINT_LOCATION_ROOT_DIR.key) } } diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexSqlITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexSqlITSuite.scala index bbd1efe41..4a1303b69 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexSqlITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexSqlITSuite.scala @@ -37,7 +37,8 @@ class FlintSparkSkippingIndexSqlITSuite extends FlintSparkSuite with ExplainSuit protected override def afterEach(): Unit = { super.afterEach() - + conf.unsetConf(FlintSparkConf.CUSTOM_FLINT_SCHEDULER_CLASS.key) + conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) deleteTestIndex(testIndex) sql(s"DROP TABLE $testTable") } @@ -98,8 +99,6 @@ class FlintSparkSkippingIndexSqlITSuite extends FlintSparkSuite with ExplainSuit flint.queryIndex(testIndex).count() shouldBe 2 sql(s"DROP SKIPPING INDEX ON $testTable") - conf.unsetConf(FlintSparkConf.CUSTOM_FLINT_SCHEDULER_CLASS.key) - conf.unsetConf(FlintSparkConf.EXTERNAL_SCHEDULER_ENABLED.key) } }