[HUDI-7465] Split tests in CI further to reduce total CI elapsed time (…

…#10795)
apache · Mar 2, 2024 · eeccdf9 · eeccdf9
1 parent f792643
commit eeccdf9
Show file tree

Hide file tree

Showing 2 changed files with 176 additions and 21 deletions.
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
@@ -53,7 +53,7 @@ jobs:
  - name: RAT check
  run: ./scripts/release/validate_source_rat.sh
 
- test-spark:
+ test-spark-java-tests:
  runs-on: ubuntu-latest
  strategy:
  matrix:
@@ -107,22 +107,87 @@ jobs:
  SPARK_PROFILE: ${{ matrix.sparkProfile }}
  run:
  mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl hudi-examples/hudi-examples-spark $MVN_ARGS
- - name: UT - Common & Spark
+ - name: Java UT - Common & Spark
  env:
  SCALA_PROFILE: ${{ matrix.scalaProfile }}
  SPARK_PROFILE: ${{ matrix.sparkProfile }}
  SPARK_MODULES: ${{ matrix.sparkModules }}
  if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
  run:
- mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
- - name: FT - Spark
+ mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ - name: Java FT - Spark
  env:
  SCALA_PROFILE: ${{ matrix.scalaProfile }}
  SPARK_PROFILE: ${{ matrix.sparkProfile }}
  SPARK_MODULES: ${{ matrix.sparkModules }}
  if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
  run:
- mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+
+ test-spark-scala-tests:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ include:
+ - scalaProfile: "scala-2.11"
+ sparkProfile: "spark2.4"
+ sparkModules: "hudi-spark-datasource/hudi-spark2"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.0"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.0.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.1"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.1.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.2"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.2.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.3"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.3.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.4"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.4.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.5"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.5.x"
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up JDK 8
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'adopt'
+ architecture: x64
+ cache: maven
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ run:
+ mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ - name: Scala UT - Common & Spark
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_MODULES: ${{ matrix.sparkModules }}
+ if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
+ run:
+ mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ - name: Scala FT - Spark
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_MODULES: ${{ matrix.sparkModules }}
+ if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
+ run:
+ mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
 
  test-hudi-hadoop-mr-and-hudi-java-client:
  runs-on: ubuntu-latest
@@ -161,7 +226,7 @@ jobs:
  run:
  ./mvnw test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS
 
- test-spark-java17:
+ test-spark-java17-java-tests:
  runs-on: ubuntu-latest
  strategy:
  matrix:
@@ -203,16 +268,16 @@ jobs:
  SCALA_PROFILE: ${{ matrix.scalaProfile }}
  SPARK_PROFILE: ${{ matrix.sparkProfile }}
  run:
- mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl hudi-examples/hudi-examples-spark $MVN_ARGS
- - name: UT - Common & Spark
+ mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl hudi-examples/hudi-examples-spark $MVN_ARGS
+ - name: Java UT - Common & Spark
  env:
  SCALA_PROFILE: ${{ matrix.scalaProfile }}
  SPARK_PROFILE: ${{ matrix.sparkProfile }}
  SPARK_MODULES: ${{ matrix.sparkModules }}
  if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
  run:
- mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
- - name: FT - Spark
+ mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ - name: Java FT - Spark
  env:
  SCALA_PROFILE: ${{ matrix.scalaProfile }}
  SPARK_PROFILE: ${{ matrix.sparkProfile }}
@@ -221,6 +286,60 @@ jobs:
  run:
  mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
 
+ test-spark-java17-scala-tests:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ include:
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.3"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.3.x"
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.4"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.4.x"
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.5"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.5.x"
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up JDK 8
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'adopt'
+ architecture: x64
+ cache: maven
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ run:
+ mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'adopt'
+ architecture: x64
+ cache: maven
+ - name: Scala UT - Common & Spark
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_MODULES: ${{ matrix.sparkModules }}
+ if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
+ run:
+ mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ - name: Scala FT - Spark
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_MODULES: ${{ matrix.sparkModules }}
+ if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
+ run:
+ mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+
  test-flink:
  runs-on: ubuntu-latest
  strategy:

diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
@@ -52,6 +52,15 @@ parameters:
  - 'hudi-spark-datasource/hudi-spark3-common'
  - 'hudi-spark-datasource/hudi-spark-common'
  - name: job4UTModules
+ type: object
+ default:
+ - 'hudi-spark-datasource'
+ - 'hudi-spark-datasource/hudi-spark'
+ - 'hudi-spark-datasource/hudi-spark3.2.x'
+ - 'hudi-spark-datasource/hudi-spark3.2plus-common'
+ - 'hudi-spark-datasource/hudi-spark3-common'
+ - 'hudi-spark-datasource/hudi-spark-common'
+ - name: job5UTModules
  type: object
  default:
  - '!hudi-hadoop-mr'
@@ -76,7 +85,7 @@ parameters:
  - '!hudi-spark-datasource/hudi-spark3.2plus-common'
  - '!hudi-spark-datasource/hudi-spark3-common'
  - '!hudi-spark-datasource/hudi-spark-common'
- - name: job4FTModules
+ - name: job5FTModules
  type: object
  default:
  - '!hudi-client/hudi-spark-client'
@@ -98,13 +107,14 @@ parameters:
 variables:
  BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.18'
  PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn'
- MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
+ MVN_OPTS_INSTALL: '-T 3 -Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
  MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)'
  JOB1_MODULES: ${{ join(',',parameters.job1Modules) }}
  JOB2_MODULES: ${{ join(',',parameters.job2Modules) }}
  JOB3_MODULES: ${{ join(',',parameters.job3UTModules) }}
- JOB4_UT_MODULES: ${{ join(',',parameters.job4UTModules) }}
- JOB4_FT_MODULES: ${{ join(',',parameters.job4FTModules) }}
+ JOB4_MODULES: ${{ join(',',parameters.job4UTModules) }}
+ JOB5_UT_MODULES: ${{ join(',',parameters.job5UTModules) }}
+ JOB5_FT_MODULES: ${{ join(',',parameters.job5FTModules) }}
 
 stages:
  - stage: test
@@ -157,7 +167,7 @@ stages:
  inputs:
  mavenPomFile: 'pom.xml'
  goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB2_MODULES) -am
  publishJUnitResults: true
  testResultsFiles: '**/surefire-reports/TEST-*.xml'
  jdkVersionOption: '1.8'
@@ -175,24 +185,24 @@ stages:
  grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
  displayName: Top 100 long-running testcases
  - job: UT_FT_3
- displayName: UT spark-datasource
+ displayName: Java UT spark-datasource
  timeoutInMinutes: '240'
  steps:
  - task: Maven@4
  displayName: maven install
  inputs:
  mavenPomFile: 'pom.xml'
  goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB3_MODULES) -am
  publishJUnitResults: true
  testResultsFiles: '**/surefire-reports/TEST-*.xml'
  jdkVersionOption: '1.8'
  - task: Maven@4
- displayName: UT spark-datasource
+ displayName: Java UT spark-datasource
  inputs:
  mavenPomFile: 'pom.xml'
  goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES)
+ options: $(MVN_OPTS_TEST) -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -Punit-tests -pl $(JOB3_MODULES)
  publishJUnitResults: true
  testResultsFiles: '**/surefire-reports/TEST-*.xml'
  jdkVersionOption: '1.8'
@@ -201,6 +211,32 @@ stages:
  grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
  displayName: Top 100 long-running testcases
  - job: UT_FT_4
+ displayName: Scala UT spark-datasource
+ timeoutInMinutes: '240'
+ steps:
+ - task: Maven@4
+ displayName: maven install
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'clean install'
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB4_MODULES) -am
+ publishJUnitResults: true
+ testResultsFiles: '**/surefire-reports/TEST-*.xml'
+ jdkVersionOption: '1.8'
+ - task: Maven@4
+ displayName: Scala UT spark-datasource
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'test'
+ options: $(MVN_OPTS_TEST) -Dtest=skipJavaTests -DfailIfNoTests=false -Punit-tests -pl $(JOB4_MODULES)
+ publishJUnitResults: true
+ testResultsFiles: '**/surefire-reports/TEST-*.xml'
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx4g'
+ - script: |
+ grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
+ displayName: Top 100 long-running testcases
+ - job: UT_FT_5
  displayName: UT FT other modules
  timeoutInMinutes: '240'
  steps:
@@ -226,6 +262,6 @@ stages:
  arguments: >
  -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
  /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source
- && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES)
- && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_UT_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB5_UT_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB5_UT_MODULES)
  && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"