Refactor integ-test path

Signed-off-by: Louis Chu <[email protected]>
opensearch-project · Jul 5, 2024 · 95f78ea · 95f78ea
1 parent b5715f6
commit 95f78ea
Show file tree

Hide file tree

Showing 44 changed files with 19 additions and 7 deletions.
diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml
@@ -23,7 +23,7 @@ jobs:
           java-version: 11
 
       - name: Integ Test
-        run: sbt integtest/test
+        run: sbt it:test
 
       - name: Unit Test
         run: sbt test

diff --git a/build.sbt b/build.sbt
@@ -196,16 +196,20 @@ lazy val flintSparkIntegration = (project in file("flint-spark-integration"))
     },
     assembly / test := (Test / test).value)
 
-// Test assembly package with integration test.
+lazy val IntegrationTest = config("it") extend Test
+
 lazy val integtest = (project in file("integ-test"))
-  .dependsOn(flintCommons % "test->test", flintSparkIntegration % "test->test", pplSparkIntegration % "test->test", sparkSqlApplication % "test->test")
+  .configs(IntegrationTest)
   .settings(
     commonSettings,
+    inConfig(IntegrationTest)(Defaults.testSettings), // Apply test settings to the IntegrationTest configuration
+    IntegrationTest / scalaSource := baseDirectory.value / "src" / "it" / "scala",
+    IntegrationTest / javaSource := baseDirectory.value / "src" / "it" / "java",
     name := "integ-test",
     scalaVersion := scala212,
     libraryDependencies ++= Seq(
       "com.amazonaws" % "aws-java-sdk" % "1.12.397" % "provided"
-        exclude ("com.fasterxml.jackson.core", "jackson-databind"),
+        exclude("com.fasterxml.jackson.core", "jackson-databind"),
       "org.scalactic" %% "scalactic" % "3.2.15",
       "org.scalatest" %% "scalatest" % "3.2.15" % "test",
       "com.stephenn" %% "scalatest-json-jsonassert" % "0.2.5" % "test",
@@ -214,11 +218,19 @@ lazy val integtest = (project in file("integ-test"))
       "org.scala-lang.modules" %% "scala-collection-compat" % "2.11.0" % "test",
       // add opensearch-java client to get node stats
       "org.opensearch.client" % "opensearch-java" % "2.6.0" % "test"
-        exclude ("com.fasterxml.jackson.core", "jackson-databind")),
+        exclude("com.fasterxml.jackson.core", "jackson-databind")),
     libraryDependencies ++= deps(sparkVersion),
-    Test / fullClasspath ++= Seq((flintSparkIntegration / assembly).value, (pplSparkIntegration / assembly).value,
+    // Adding assembled jars to the classpath
+    Test / fullClasspath ++= Seq(
+      (flintSparkIntegration / assembly).value,
+      (pplSparkIntegration / assembly).value,
       (sparkSqlApplication / assembly).value
-    ))
+    ),
+    // Make sure integration tests do not run during the normal test phase
+    Test / test := (),
+    IntegrationTest / test := (IntegrationTest / test).dependsOn(Compile / compile).value
+  )
+  .dependsOn(flintCommons % "test->test", flintSparkIntegration % "test->test", pplSparkIntegration % "test->test", sparkSqlApplication % "test->test")
 
 lazy val standaloneCosmetic = project
   .settings(

diff --git a/...h/testcontainers/OpenSearchContainer.java → ...h/testcontainers/OpenSearchContainer.java b/...h/testcontainers/OpenSearchContainer.java → ...h/testcontainers/OpenSearchContainer.java
diff --git a/...ache/spark/FlintDataSourceV2ITSuite.scala → ...ache/spark/FlintDataSourceV2ITSuite.scala b/...ache/spark/FlintDataSourceV2ITSuite.scala → ...ache/spark/FlintDataSourceV2ITSuite.scala
diff --git a/...ch/catalog/OpenSearchCatalogITSuite.scala → ...ch/catalog/OpenSearchCatalogITSuite.scala b/...ch/catalog/OpenSearchCatalogITSuite.scala → ...ch/catalog/OpenSearchCatalogITSuite.scala
diff --git a/...rg/apache/spark/sql/FlintJobITSuite.scala → ...rg/apache/spark/sql/FlintJobITSuite.scala b/...rg/apache/spark/sql/FlintJobITSuite.scala → ...rg/apache/spark/sql/FlintJobITSuite.scala
diff --git a/...g/apache/spark/sql/FlintREPLITSuite.scala → ...g/apache/spark/sql/FlintREPLITSuite.scala b/...g/apache/spark/sql/FlintREPLITSuite.scala → ...g/apache/spark/sql/FlintREPLITSuite.scala
diff --git a/.../scala/org/apache/spark/sql/JobTest.scala → .../scala/org/apache/spark/sql/JobTest.scala b/.../scala/org/apache/spark/sql/JobTest.scala → .../scala/org/apache/spark/sql/JobTest.scala
diff --git a/...ala/org/apache/spark/sql/REPLResult.scala → ...ala/org/apache/spark/sql/REPLResult.scala b/...ala/org/apache/spark/sql/REPLResult.scala → ...ala/org/apache/spark/sql/REPLResult.scala
diff --git a/...che/spark/sql/SparkHiveSupportSuite.scala → ...che/spark/sql/SparkHiveSupportSuite.scala b/...che/spark/sql/SparkHiveSupportSuite.scala → ...che/spark/sql/SparkHiveSupportSuite.scala
diff --git a/...rg/opensearch/flint/OpenSearchSuite.scala → ...rg/opensearch/flint/OpenSearchSuite.scala b/...rg/opensearch/flint/OpenSearchSuite.scala → ...rg/opensearch/flint/OpenSearchSuite.scala
diff --git a/...ch/flint/OpenSearchTransactionSuite.scala → ...ch/flint/OpenSearchTransactionSuite.scala b/...ch/flint/OpenSearchTransactionSuite.scala → ...ch/flint/OpenSearchTransactionSuite.scala
diff --git a/.../flint/core/FlintMetadataLogITSuite.scala → .../flint/core/FlintMetadataLogITSuite.scala b/.../flint/core/FlintMetadataLogITSuite.scala → .../flint/core/FlintMetadataLogITSuite.scala
diff --git a/...int/core/FlintOpenSearchClientSuite.scala → ...int/core/FlintOpenSearchClientSuite.scala b/...int/core/FlintOpenSearchClientSuite.scala → ...int/core/FlintOpenSearchClientSuite.scala
diff --git a/.../flint/core/FlintTransactionITSuite.scala → .../flint/core/FlintTransactionITSuite.scala b/.../flint/core/FlintTransactionITSuite.scala → .../flint/core/FlintTransactionITSuite.scala
diff --git a/...h/flint/core/OpenSearchUpdaterSuite.scala → ...h/flint/core/OpenSearchUpdaterSuite.scala b/...h/flint/core/OpenSearchUpdaterSuite.scala → ...h/flint/core/OpenSearchUpdaterSuite.scala
diff --git a/...park/FlintSparkCoveringIndexITSuite.scala → ...park/FlintSparkCoveringIndexITSuite.scala b/...park/FlintSparkCoveringIndexITSuite.scala → ...park/FlintSparkCoveringIndexITSuite.scala
diff --git a/...k/FlintSparkCoveringIndexSqlITSuite.scala → ...k/FlintSparkCoveringIndexSqlITSuite.scala b/...k/FlintSparkCoveringIndexSqlITSuite.scala → ...k/FlintSparkCoveringIndexSqlITSuite.scala
diff --git a/...int/spark/FlintSparkIndexJobITSuite.scala → ...int/spark/FlintSparkIndexJobITSuite.scala b/...int/spark/FlintSparkIndexJobITSuite.scala → ...int/spark/FlintSparkIndexJobITSuite.scala
diff --git a/.../spark/FlintSparkIndexJobSqlITSuite.scala → .../spark/FlintSparkIndexJobSqlITSuite.scala b/.../spark/FlintSparkIndexJobSqlITSuite.scala → .../spark/FlintSparkIndexJobSqlITSuite.scala
diff --git a/...spark/FlintSparkIndexMonitorITSuite.scala → ...spark/FlintSparkIndexMonitorITSuite.scala b/...spark/FlintSparkIndexMonitorITSuite.scala → ...spark/FlintSparkIndexMonitorITSuite.scala
diff --git a/...nt/spark/FlintSparkIndexNameITSuite.scala → ...nt/spark/FlintSparkIndexNameITSuite.scala b/...nt/spark/FlintSparkIndexNameITSuite.scala → ...nt/spark/FlintSparkIndexNameITSuite.scala
diff --git a/...int/spark/FlintSparkIndexSqlITSuite.scala → ...int/spark/FlintSparkIndexSqlITSuite.scala b/...int/spark/FlintSparkIndexSqlITSuite.scala → ...int/spark/FlintSparkIndexSqlITSuite.scala
diff --git a/...rk/FlintSparkIndexValidationITSuite.scala → ...rk/FlintSparkIndexValidationITSuite.scala b/...rk/FlintSparkIndexValidationITSuite.scala → ...rk/FlintSparkIndexValidationITSuite.scala
diff --git a/...k/FlintSparkMaterializedViewITSuite.scala → ...k/FlintSparkMaterializedViewITSuite.scala b/...k/FlintSparkMaterializedViewITSuite.scala → ...k/FlintSparkMaterializedViewITSuite.scala
diff --git a/...lintSparkMaterializedViewSqlITSuite.scala → ...lintSparkMaterializedViewSqlITSuite.scala b/...lintSparkMaterializedViewSqlITSuite.scala → ...lintSparkMaterializedViewSqlITSuite.scala
diff --git a/...park/FlintSparkSkippingIndexITSuite.scala → ...park/FlintSparkSkippingIndexITSuite.scala b/...park/FlintSparkSkippingIndexITSuite.scala → ...park/FlintSparkSkippingIndexITSuite.scala
diff --git a/...k/FlintSparkSkippingIndexSqlITSuite.scala → ...k/FlintSparkSkippingIndexSqlITSuite.scala b/...k/FlintSparkSkippingIndexSqlITSuite.scala → ...k/FlintSparkSkippingIndexSqlITSuite.scala
diff --git a/...nsearch/flint/spark/FlintSparkSuite.scala → ...nsearch/flint/spark/FlintSparkSuite.scala b/...nsearch/flint/spark/FlintSparkSuite.scala → ...nsearch/flint/spark/FlintSparkSuite.scala
diff --git a/.../spark/FlintSparkTransactionITSuite.scala → .../spark/FlintSparkTransactionITSuite.scala b/.../spark/FlintSparkTransactionITSuite.scala → .../spark/FlintSparkTransactionITSuite.scala
diff --git a/.../spark/FlintSparkUpdateIndexITSuite.scala → .../spark/FlintSparkUpdateIndexITSuite.scala b/.../spark/FlintSparkUpdateIndexITSuite.scala → .../spark/FlintSparkUpdateIndexITSuite.scala
diff --git a/.../FlintSparkWindowingFunctionITSuite.scala → .../FlintSparkWindowingFunctionITSuite.scala b/.../FlintSparkWindowingFunctionITSuite.scala → .../FlintSparkWindowingFunctionITSuite.scala
diff --git a/...ch/flint/spark/LogicalPlanTestUtils.scala → ...ch/flint/spark/LogicalPlanTestUtils.scala b/...ch/flint/spark/LogicalPlanTestUtils.scala → ...ch/flint/spark/LogicalPlanTestUtils.scala
diff --git a/...intSparkIcebergCoveringIndexITSuite.scala → ...intSparkIcebergCoveringIndexITSuite.scala b/...intSparkIcebergCoveringIndexITSuite.scala → ...intSparkIcebergCoveringIndexITSuite.scala
diff --git a/...SparkIcebergMaterializedViewITSuite.scala → ...SparkIcebergMaterializedViewITSuite.scala b/...SparkIcebergMaterializedViewITSuite.scala → ...SparkIcebergMaterializedViewITSuite.scala
diff --git a/...intSparkIcebergSkippingIndexITSuite.scala → ...intSparkIcebergSkippingIndexITSuite.scala b/...intSparkIcebergSkippingIndexITSuite.scala → ...intSparkIcebergSkippingIndexITSuite.scala
diff --git a/...park/iceberg/FlintSparkIcebergSuite.scala → ...park/iceberg/FlintSparkIcebergSuite.scala b/...park/iceberg/FlintSparkIcebergSuite.scala → ...park/iceberg/FlintSparkIcebergSuite.scala
diff --git a/...earch/flint/spark/ppl/FlintPPLSuite.scala → ...earch/flint/spark/ppl/FlintPPLSuite.scala b/...earch/flint/spark/ppl/FlintPPLSuite.scala → ...earch/flint/spark/ppl/FlintPPLSuite.scala
diff --git a/...tSparkPPLAggregationWithSpanITSuite.scala → ...tSparkPPLAggregationWithSpanITSuite.scala b/...tSparkPPLAggregationWithSpanITSuite.scala → ...tSparkPPLAggregationWithSpanITSuite.scala
diff --git a/...pl/FlintSparkPPLAggregationsITSuite.scala → ...pl/FlintSparkPPLAggregationsITSuite.scala b/...pl/FlintSparkPPLAggregationsITSuite.scala → ...pl/FlintSparkPPLAggregationsITSuite.scala
diff --git a/...spark/ppl/FlintSparkPPLBasicITSuite.scala → ...spark/ppl/FlintSparkPPLBasicITSuite.scala b/...spark/ppl/FlintSparkPPLBasicITSuite.scala → ...spark/ppl/FlintSparkPPLBasicITSuite.scala
diff --git a/...ppl/FlintSparkPPLCorrelationITSuite.scala → ...ppl/FlintSparkPPLCorrelationITSuite.scala b/...ppl/FlintSparkPPLCorrelationITSuite.scala → ...ppl/FlintSparkPPLCorrelationITSuite.scala
diff --git a/...ark/ppl/FlintSparkPPLFiltersITSuite.scala → ...ark/ppl/FlintSparkPPLFiltersITSuite.scala b/...ark/ppl/FlintSparkPPLFiltersITSuite.scala → ...ark/ppl/FlintSparkPPLFiltersITSuite.scala
diff --git a/.../ppl/FlintSparkPPLTimeWindowITSuite.scala → .../ppl/FlintSparkPPLTimeWindowITSuite.scala b/.../ppl/FlintSparkPPLTimeWindowITSuite.scala → .../ppl/FlintSparkPPLTimeWindowITSuite.scala