Split integration test to multiple groups and enable parallelExecution

Signed-off-by: Heng Qian <[email protected]>

build.sbt

@@ -2,8 +2,7 @@
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
  */
-import Dependencies._
-import sbtassembly.AssemblyPlugin.autoImport.ShadeRule
+import Dependencies.*
 
 lazy val scala212 = "2.12.14"
 lazy val sparkVersion = "3.5.1"
@@ -38,6 +37,11 @@ ThisBuild / scalastyleConfig := baseDirectory.value / "scalastyle-config.xml"
  */
 ThisBuild / Test / parallelExecution := false
 
+/**
+ * Set the parallelism of forked tests to 4 to accelerate integration test
+ */
+concurrentRestrictions in Global := Seq(Tags.limit(Tags.ForkedTestGroup, 4))
+
 // Run as part of compile task.
 lazy val compileScalastyle = taskKey[Unit]("compileScalastyle")
 
@@ -274,13 +278,29 @@ lazy val integtest = (project in file("integ-test"))
       IntegrationTest / javaSource := baseDirectory.value / "src/integration/java",
       IntegrationTest / scalaSource := baseDirectory.value / "src/integration/scala",
       IntegrationTest / resourceDirectory := baseDirectory.value / "src/integration/resources",
-        IntegrationTest / parallelExecution := false,
+      IntegrationTest / parallelExecution := true, // enable parallel execution
+      IntegrationTest / testForkedParallel := false, // disable forked parallel execution to avoid duplicate spark context in the same JVM
       IntegrationTest / fork := true,
+      IntegrationTest / testGrouping := {
+        val tests = (IntegrationTest / definedTests).value
+        val forkOptions = ForkOptions()
+        val groups = tests.grouped(tests.size / 4 + 1).zipWithIndex.map { case (group, index) =>
+          val groupName = s"group-${index + 1}"
+          new Tests.Group(
+            name = groupName,
+            tests = group,
+            runPolicy = Tests.SubProcess(
+              forkOptions.withRunJVMOptions(forkOptions.runJVMOptions ++
+                Seq(s"-Djava.io.tmpdir=${baseDirectory.value}/integ-test/target/tmp/$groupName")))
+          )
+        }
+        groups.toSeq
+      }
     )),
     inConfig(AwsIntegrationTest)(Defaults.testSettings ++ Seq(
       AwsIntegrationTest / javaSource := baseDirectory.value / "src/aws-integration/java",
       AwsIntegrationTest / scalaSource := baseDirectory.value / "src/aws-integration/scala",
-      AwsIntegrationTest / parallelExecution := false,
+      AwsIntegrationTest / parallelExecution := true,
       AwsIntegrationTest / fork := true,
     )),
     libraryDependencies ++= Seq(