polish ci and extract base ci

.github/workflows/flink_cdc.yml

@@ -46,16 +46,54 @@ concurrency:
 jobs:
   license_check:
     name: License Check
-    uses: ./.github/workflows/license_check.yml
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Set up Ruby environment
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+      - name: Compiling jar packages
+        run: mvn --no-snapshot-updates -B package -DskipTests
+      - name: Run license check
+        run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
   ut:
+    strategy:
+      matrix:
+        java-version: [ '8' ]
+        module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
     name: Unit Tests
-    uses: ./.github/workflows/flink_cdc_ut.yml
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-version: ${{ matrix.java-version }}
+      module: ${{ matrix.module }}
   pipeline_e2e:
+    strategy:
+      matrix:
+        java-version: [ '8' ]
+        flink-version: ['1.17.2', '1.18.1', '1.19.1', '1.20.0']
+        module: [ 'pipeline_e2e' ]
     name: Pipeline End-to-end Tests
-    uses: ./.github/workflows/flink_cdc_pipeline_e2e.yml
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-version: ${{ matrix.java-version }}
+      flink-version: ${{ matrix.flink-version }}
+      module: ${{ matrix.module }}
   source_e2e:
+    strategy:
+      matrix:
+        java-version: [ '8' ]
+        flink-version: ['1.16.3', '1.17.2', '1.18.1', '1.19.1', '1.20.0']
+        module: [ 'source_e2e' ]
     name: Source End-to-end Tests
-    uses: ./.github/workflows/flink_cdc_source_e2e.yml
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-version: ${{ matrix.java-version }}
+      flink-version: ${{ matrix.flink-version }}
+      module: ${{ matrix.module }}
   migration_test:
     name: Migration Tests
     uses: ./.github/workflows/flink_cdc_migration_test.yml

.github/workflows/flink_cdc_ut.yml → .github/workflows/flink_cdc_base.yml

@@ -13,10 +13,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Unit Tests
+name: Flink CDC Base Workflow
 
 on:
   workflow_call:
+    inputs:
+      java-version:
+        description: "Jdk version to test against."
+        required: false
+        type: string
+        default: "8"
+      flink-version:
+        description: "Flink version to test against."
+        required: false
+        type: string
+      module:
+        description: "Flink CDC module to test against."
+        required: true
+        type: string
+      custom-maven-parameter:
+        description: "Custom maven parameter."
+        required: false
+        type: string
 
 env:
   MODULES_CORE: "\
@@ -76,24 +94,8 @@ env:
   flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
 
 jobs:
-  flink_cdc_ut:
+  compile_and_test:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        jdk: [ 8 ]
-        module: [ "core",
-                  "pipeline_connectors",
-                  "mysql",
-                  "postgres",
-                  "oracle",
-                  "mongodb6",
-                  "mongodb7",
-                  "sqlserver",
-                  "tidb",
-                  "oceanbase",
-                  "db2",
-                  "vitess"
-        ]
     timeout-minutes: 120
     env:
       MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \
@@ -105,7 +107,7 @@ jobs:
         -Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \
         -Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties
     steps:
-      - run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}"
+      - run: echo "Running CI pipeline for JDK version ${{ inputs.java-version }}"
 
       - name: Clean up disk space
         run: |
@@ -132,7 +134,7 @@ jobs:
       - name: Set JDK
         uses: actions/setup-java@v4
         with:
-          java-version: ${{ matrix.jdk }}
+          java-version: ${{ inputs.java-version }}
           distribution: 'temurin'
           cache: 'maven'
 
@@ -141,12 +143,12 @@ jobs:
         with:
           maven-version: 3.8.6
 
-      - name: Compile and test ${{ matrix.module }}
-        timeout-minutes: 90
+      - name: Compile and test
+        timeout-minutes: 60
         run: |
           set -o pipefail
 
-          case ${{ matrix.module }} in
+          case ${{ inputs.module }} in
               ("core")
                modules=${{ env.MODULES_CORE }}
               ;;
@@ -183,34 +185,42 @@ jobs:
               ("vitess")
                modules=${{ env.MODULES_VITESS }}
               ;;
+              ("pipeline_e2e")
+               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
+               modules=${{ env.MODULES_PIPELINE_E2E }}
+              ;;
+              ("source_e2e")
+               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
+               modules=${{ env.MODULES_SOURCE_E2E }}
+              ;;
           esac
+          
+          if [ ${{ inputs.module }} != "pipeline_e2e" ] && [ ${{ inputs.module }} != "source_e2e" ]; then
+            compile_modules=$modules
+          fi
+          
+          custom_maven_parameter=""
 
-          if [[ ${{ matrix.module }} == "mongodb6" ]]; then
-            specifiedMongoVersion="-DspecifiedMongoVersion=6.0.16"
-          elif [[ ${{ matrix.module }} == "mongodb7" ]]; then
-            specifiedMongoVersion="-DspecifiedMongoVersion=7.0.12"
-          else
-            specifiedMongoVersion=""
+          if [ ${{ inputs.module }} == "mongodb6" ]; then
+            custom_maven_parameter="-DspecifiedMongoVersion=6.0.16"
+          elif [ ${{ inputs.module }} == "mongodb7" ]; then
+            custom_maven_parameter="-DspecifiedMongoVersion=7.0.12"
+          fi
+          
+          if [ -z "${{ inputs.flink-version }}" ]; then
+            custom_maven_parameter="${custom_maven_parameter} -DspecifiedFlinkVersion=${{ inputs.flink-version }}"
           fi
           
-          mvn --no-snapshot-updates -B -DskipTests -pl $modules -am install && mvn --no-snapshot-updates -B $specifiedMongoVersion -pl $modules verify
+          mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $custom_maven_parameter -pl $modules verify
 
       - name: Print JVM thread dumps when cancelled
         if: ${{ failure() }}
         run: |
-          # ----------------------------------------------------------------------------
-          # Copyright 2023 The Netty Project
-          #
-          # ----------------------------------------------------------------------------
-          # Source: https://github.com/netty/netty/blob/main/.github/actions/thread-dump-jvms/action.yml
           echo "$OSTYPE"
           if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then
             echo "Setting up JVM thread dumps"
-            # use jattach so that Java processes in docker containers are also covered
-            # download jattach
             curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach
             if command -v sha256sum &> /dev/null; then
-              # verify hash of jattach binary
               sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e  /tmp/jattach") || exit 1
             fi
             chmod +x /tmp/jattach