diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index 3c06acb61..b19c54428 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -23,7 +23,7 @@ jobs: java-version: 11 - name: Integ Test - run: sbt integtest/test + run: sbt it:test - name: Unit Test run: sbt test diff --git a/build.sbt b/build.sbt index b04aad2ee..7ff027f8c 100644 --- a/build.sbt +++ b/build.sbt @@ -196,16 +196,20 @@ lazy val flintSparkIntegration = (project in file("flint-spark-integration")) }, assembly / test := (Test / test).value) -// Test assembly package with integration test. +lazy val IntegrationTest = config("it") extend Test + lazy val integtest = (project in file("integ-test")) - .dependsOn(flintCommons % "test->test", flintSparkIntegration % "test->test", pplSparkIntegration % "test->test", sparkSqlApplication % "test->test") + .configs(IntegrationTest) .settings( commonSettings, + inConfig(IntegrationTest)(Defaults.testSettings), // Apply test settings to the IntegrationTest configuration + IntegrationTest / scalaSource := baseDirectory.value / "src" / "it" / "scala", + IntegrationTest / javaSource := baseDirectory.value / "src" / "it" / "java", name := "integ-test", scalaVersion := scala212, libraryDependencies ++= Seq( "com.amazonaws" % "aws-java-sdk" % "1.12.397" % "provided" - exclude ("com.fasterxml.jackson.core", "jackson-databind"), + exclude("com.fasterxml.jackson.core", "jackson-databind"), "org.scalactic" %% "scalactic" % "3.2.15", "org.scalatest" %% "scalatest" % "3.2.15" % "test", "com.stephenn" %% "scalatest-json-jsonassert" % "0.2.5" % "test", @@ -214,11 +218,19 @@ lazy val integtest = (project in file("integ-test")) "org.scala-lang.modules" %% "scala-collection-compat" % "2.11.0" % "test", // add opensearch-java client to get node stats "org.opensearch.client" % "opensearch-java" % "2.6.0" % "test" - exclude ("com.fasterxml.jackson.core", "jackson-databind")), + exclude("com.fasterxml.jackson.core", "jackson-databind")), libraryDependencies ++= deps(sparkVersion), - Test / fullClasspath ++= Seq((flintSparkIntegration / assembly).value, (pplSparkIntegration / assembly).value, + // Adding assembled jars to the classpath + Test / fullClasspath ++= Seq( + (flintSparkIntegration / assembly).value, + (pplSparkIntegration / assembly).value, (sparkSqlApplication / assembly).value - )) + ), + // Make sure integration tests do not run during the normal test phase + Test / test := (), + IntegrationTest / test := (IntegrationTest / test).dependsOn(Compile / compile).value + ) + .dependsOn(flintCommons % "test->test", flintSparkIntegration % "test->test", pplSparkIntegration % "test->test", sparkSqlApplication % "test->test") lazy val standaloneCosmetic = project .settings( diff --git a/integ-test/src/test/java/org/opensearch/testcontainers/OpenSearchContainer.java b/integ-test/src/it/java/org/opensearch/testcontainers/OpenSearchContainer.java similarity index 100% rename from integ-test/src/test/java/org/opensearch/testcontainers/OpenSearchContainer.java rename to integ-test/src/it/java/org/opensearch/testcontainers/OpenSearchContainer.java diff --git a/integ-test/src/test/scala/org/apache/spark/FlintDataSourceV2ITSuite.scala b/integ-test/src/it/scala/org/apache/spark/FlintDataSourceV2ITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/apache/spark/FlintDataSourceV2ITSuite.scala rename to integ-test/src/it/scala/org/apache/spark/FlintDataSourceV2ITSuite.scala diff --git a/integ-test/src/test/scala/org/apache/spark/opensearch/catalog/OpenSearchCatalogITSuite.scala b/integ-test/src/it/scala/org/apache/spark/opensearch/catalog/OpenSearchCatalogITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/apache/spark/opensearch/catalog/OpenSearchCatalogITSuite.scala rename to integ-test/src/it/scala/org/apache/spark/opensearch/catalog/OpenSearchCatalogITSuite.scala diff --git a/integ-test/src/test/scala/org/apache/spark/sql/FlintJobITSuite.scala b/integ-test/src/it/scala/org/apache/spark/sql/FlintJobITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/apache/spark/sql/FlintJobITSuite.scala rename to integ-test/src/it/scala/org/apache/spark/sql/FlintJobITSuite.scala diff --git a/integ-test/src/test/scala/org/apache/spark/sql/FlintREPLITSuite.scala b/integ-test/src/it/scala/org/apache/spark/sql/FlintREPLITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/apache/spark/sql/FlintREPLITSuite.scala rename to integ-test/src/it/scala/org/apache/spark/sql/FlintREPLITSuite.scala diff --git a/integ-test/src/test/scala/org/apache/spark/sql/JobTest.scala b/integ-test/src/it/scala/org/apache/spark/sql/JobTest.scala similarity index 100% rename from integ-test/src/test/scala/org/apache/spark/sql/JobTest.scala rename to integ-test/src/it/scala/org/apache/spark/sql/JobTest.scala diff --git a/integ-test/src/test/scala/org/apache/spark/sql/REPLResult.scala b/integ-test/src/it/scala/org/apache/spark/sql/REPLResult.scala similarity index 100% rename from integ-test/src/test/scala/org/apache/spark/sql/REPLResult.scala rename to integ-test/src/it/scala/org/apache/spark/sql/REPLResult.scala diff --git a/integ-test/src/test/scala/org/apache/spark/sql/SparkHiveSupportSuite.scala b/integ-test/src/it/scala/org/apache/spark/sql/SparkHiveSupportSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/apache/spark/sql/SparkHiveSupportSuite.scala rename to integ-test/src/it/scala/org/apache/spark/sql/SparkHiveSupportSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/OpenSearchSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/OpenSearchSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/OpenSearchSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/OpenSearchSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/OpenSearchTransactionSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/OpenSearchTransactionSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/OpenSearchTransactionSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/OpenSearchTransactionSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/core/FlintMetadataLogITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/core/FlintMetadataLogITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/core/FlintMetadataLogITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/core/FlintMetadataLogITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/core/FlintOpenSearchClientSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/core/FlintOpenSearchClientSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/core/FlintOpenSearchClientSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/core/FlintOpenSearchClientSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/core/FlintTransactionITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/core/FlintTransactionITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/core/FlintTransactionITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/core/FlintTransactionITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/core/OpenSearchUpdaterSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/core/OpenSearchUpdaterSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/core/OpenSearchUpdaterSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/core/OpenSearchUpdaterSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexSqlITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexSqlITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexSqlITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexSqlITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexJobITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexJobITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexJobITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexJobITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexJobSqlITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexJobSqlITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexJobSqlITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexJobSqlITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexMonitorITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexMonitorITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexMonitorITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexMonitorITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexNameITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexNameITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexNameITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexNameITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexSqlITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexSqlITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexSqlITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexSqlITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexValidationITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexValidationITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexValidationITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkIndexValidationITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexSqlITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexSqlITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexSqlITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexSqlITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkTransactionITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkTransactionITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkTransactionITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkTransactionITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkUpdateIndexITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkUpdateIndexITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkUpdateIndexITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkUpdateIndexITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkWindowingFunctionITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkWindowingFunctionITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkWindowingFunctionITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/FlintSparkWindowingFunctionITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/LogicalPlanTestUtils.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/LogicalPlanTestUtils.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/LogicalPlanTestUtils.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/LogicalPlanTestUtils.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergCoveringIndexITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergCoveringIndexITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergCoveringIndexITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergCoveringIndexITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergMaterializedViewITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergMaterializedViewITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergMaterializedViewITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergMaterializedViewITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergSkippingIndexITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergSkippingIndexITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergSkippingIndexITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergSkippingIndexITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/iceberg/FlintSparkIcebergSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintPPLSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintPPLSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintPPLSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintPPLSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLAggregationWithSpanITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLAggregationWithSpanITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLAggregationWithSpanITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLAggregationWithSpanITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLAggregationsITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLAggregationsITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLAggregationsITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLAggregationsITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBasicITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBasicITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBasicITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBasicITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLCorrelationITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLCorrelationITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLCorrelationITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLCorrelationITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLFiltersITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLFiltersITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLFiltersITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLFiltersITSuite.scala diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLTimeWindowITSuite.scala b/integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLTimeWindowITSuite.scala similarity index 100% rename from integ-test/src/test/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLTimeWindowITSuite.scala rename to integ-test/src/it/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLTimeWindowITSuite.scala