debezium-workflow-pr.yml

name: Build Debezium (PR)

on:
  pull_request:
    branches:
      - main
      - 1.*
      - 2.*

env:
  MAVEN_FULL_BUILD_PROJECTS: "\\!debezium-microbenchmark-oracle"

# Pushes to each pull request will trigger a cancellation of any existing push jobs for that pull request,
# and will restart the build based on the latest push data for that specific pull request.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}
  cancel-in-progress: true

jobs:
  # This job is responsible for inspecting the changes in the repository and setting specific output variables
  # that subsequent jobs can use to determine whether a particular job should be skipped or included.
  file_changes:
    name: "Detect repository changes"
    runs-on: ubuntu-latest
    outputs:
      common-changed: ${{ steps.changed-files-common.outputs.any_changed }}
      mongodb-changed: ${{ steps.changed-files-mongodb.outputs.any_changed }}
      mariadb-changed: ${{ steps.changed-files-mariadb.outputs.any_changed }}
      mysql-changed: ${{ steps.changed-files-mysql.outputs.any_changed }}
      postgresql-changed: ${{ steps.changed-files-postgresql.outputs.any_changed }}
      oracle-changed: ${{ steps.changed-files-oracle.outputs.any_changed }}
      sqlserver-changed: ${{ steps.changed-files-sqlserver.outputs.any_changed }}
      outbox-changed: ${{ steps.changed-files-outbox.outputs.any_changed }}
      rest-extension-changed: ${{ steps.changed-files-rest-extension.outputs.any_changed }}
      schema-generator-changed: ${{ steps.changed-files-schema-generator.outputs.any_changed }}
      debezium-testing-changed: ${{ steps.changed-files-debezium-testing.outputs.any_changed }}
      debezium-testing-mongodb-changed: ${{ steps.changed-files-debezium-testing-mongodb.outputs.any_changed }}
      mysql-ddl-parser-changed: ${{ steps.changed-files-mysql-ddl-parser.outputs.any_changed }}
      oracle-ddl-parser-changed: ${{ steps.changed-files-oracle-ddl-parser.outputs.any_changed }}
      mariadb-ddl-parser-changed: ${{ steps.changed-files-mariadb-ddl-parser.outputs.any_changed }}
      documentation-only-changed: ${{ steps.changed-files-documentation.outputs.only_changed }}
      storage-only-changed: ${{ steps.changed-files-storage.outputs.only_changed }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Get modified files (Common)
        id: changed-files-common
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            support/checkstyle/**
            support/revapi/**
            debezium-api/**
            debezium-assembly-descriptors/**
            debezium-core/**
            debezium-embedded/**
            debezium-revapi/**
            debezium-ide-configs/**
            debezium-parent/pom.xml
            debezium-bom/pom.xml
            debezium-storage/** 
            debezium-scripting/**          
            pom.xml
            .mvn/**
            .github/actions/**
            .github/workflows/debezium-workflow-pr.yml

      - name: Get modified files (MongoDB)
        id: changed-files-mongodb
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-connector-mongodb/**

      - name: Get modified files (MySQL)
        id: changed-files-mysql
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-connector-mysql/**

      - name: Get modified files (MariaDB)
        id: changed-files-mariadb
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-connector-mariadb/**
            debezium-connector-binlog/**

      - name: Get modified files (PostgreSQL)
        id: changed-files-postgresql
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-connector-postgres/**

      - name: Get modified files (Oracle)
        id: changed-files-oracle
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-connector-oracle/**

      - name: Get modified files (SQL Server)
        id: changed-files-sqlserver
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-connector-sqlserver/**

      - name: Get modified files (Quarkus Outbox)
        id: changed-files-outbox
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-quarkus-outbox/**
            debezium-quarkus-outbox-common/**
            debezium-quarkus-outbox-reactive/**

      - name: Get modified files (REST Extension)
        id: changed-files-rest-extension
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-connect-rest-extension/**

      - name: Get modified files (Schema Generator)
        id: changed-files-schema-generator
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-schema-generator/**

      - name: Get modified files (Debezium Testing)
        id: changed-files-debezium-testing
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-testing/**

      - name: Get modified files (Debezium Testing MongoDB)
        id: changed-files-debezium-testing-mongodb
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-testing/**/MongoDb*.java

      - name: Get modified files (MySQL DDL parser)
        id: changed-files-mysql-ddl-parser
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-ddl-parser/src/main/antlr4/io/debezium/ddl/parser/mysql/**
            debezium-ddl-parser/src/main/java/io/debezium/antlr/**
            debezium-ddl-parser/src/test/resources/mysql/examples/**

      - name: Get modified files (Oracle DDL parser)
        id: changed-files-oracle-ddl-parser
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-ddl-parser/src/main/antlr4/io/debezium/ddl/parser/oracle/**
            debezium-ddl-parser/src/main/java/io/debezium/antlr/**
            debezium-ddl-parser/src/main/java/io/debezium/ddl/parser/oracle/**
            debezium-ddl-parser/src/test/resources/oracle/examples/**

      - name: Get modified files (Documentation)
        id: changed-files-documentation
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            documentation/**

      - name: Get modified files (Storage)
        id: changed-files-storage
        uses: tj-actions/changed-files@v44.5.5
        with:
          files: |
            debezium-storage/**

  # Approx 1m
  build_cache:
    name: "Update Dependencies"
    needs: [ file_changes ]
    if: ${{ needs.file_changes.outputs.documentation-only-changed == 'false' }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Action (Core)
        uses: actions/checkout@v4
        with:
          path: core

      - name: Checkout Action (Debezium Server)
        uses: actions/checkout@v4
        with:
          repository: debezium/debezium-server
          path: server
          ref: ${{ github.event.pull_request.base.ref }} 

      - uses: ./core/.github/actions/setup-java

      # GitHub actions seem to struggle returning actions/cache cache-hit
      # Directly use the cache action here to control whether to fetch dependencies
      - id: maven-cache-check
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
          restore-keys: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}

      # This step is responsible for pulling down artifacts
      # Unfortunately due to the nature of how some of the maven workflows work, the only reliable way
      # to guarantee fully seeding the maven cache is to run a full build. This step does not execute
      # tests, formatting, checkstyle, nor import sorts.
      #
      # This check is conditioned so that if the cache-key was not found, we will execute this step.
      # If the cache-key was found, this means we cannot update the cache and therefore we should
      # not need to explicitly run this step. This should improve response of format/checkstyle
      # errors to users faster.
      #
      # This job also explicitly excludes the "debezium-microbenchmark-oracle" module temporarily.
      # There is a dependency on xstream.jar for this module that should be fixed and made to not
      # be required so that the module can be built on GitHub Actions.
      - name: Download dependencies (Core)
        if: steps.maven-cache-check.outputs.cache-hit != 'true'
        run: >
          ./core/mvnw -B -ntp clean install -f core/pom.xml
          -pl ${{ env.MAVEN_FULL_BUILD_PROJECTS }}
          -Dformat.skip=true
          -Dcheckstyle.skip=true
          -Dorg.slf4j.simpleLogger.showDateTime=true
          -Dorg.slf4j.simpleLogger.dateTimeFormat="YYYY-MM-dd HH:mm:ss,SSS"
          -DskipTests=true
          -DskipITs=true

      - name: Download dependencies (Debezium Server)
        if: steps.maven-cache-check.outputs.cache-hit != 'true'
        run: >
          ./server/mvnw -B -ntp clean install -f server/pom.xml
          -Dformat.skip=true
          -Dcheckstyle.skip=true
          -Dorg.slf4j.simpleLogger.showDateTime=true
          -Dorg.slf4j.simpleLogger.dateTimeFormat="YYYY-MM-dd HH:mm:ss,SSS"
          -DskipTests=true
          -DskipITs=true

  # Approx 1m
  check_style:
    name: "Checkstyle and Formatting"
    needs: [ build_cache ]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/checkstyle-format
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  build_mongodb:
    strategy:
      # Runs each combination concurrently
      matrix:
        version-mongo-server: [ "5.0", "7.0" ]
      fail-fast: false
    name: "MongoDB ${{ matrix.version-mongo-server }}"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mongodb-changed == 'true' || needs.file_changes.outputs.debezium-testing-mongodb-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-mongodb
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
          version-mongo-server: ${{ matrix.version-mongo-server }}

  # Approx 40m each
  build_mysql:
    strategy:
      # Runs each combination concurrently
      matrix:
        profile: [ "mysql-ci", "mysql-ci-gtids", "mysql-ci-percona", "mysql-ci-ssl" ]
        version-mysql-server: [ "8.0", "8.2" ]
      fail-fast: false
    name: "MySQL ${{ matrix.version-mysql-server }} - ${{ matrix.profile }}"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-mysql
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
          version-mysql-server: ${{ matrix.version-mysql-server }}
          profile: ${{ matrix.profile }}

  # Approx 40m each
  build_mariadb:
    strategy:
      # Runs each combination concurrently
      matrix:
        profile: [ 'mariadb-ci', 'mariadb-ci-gtids' ]
      fail-fast: false
    name: "MariaDB - ${{ matrix.profile }}"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mariadb-changed == 'true' || needs.file_changes.outputs.mariadb-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-mariadb
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
          profile: ${{ matrix.profile }}

  # Approx 40m each
  build_postgresql:
    strategy:
      # Runs each combination concurrently
      matrix:
        profile: [ "assembly", "assembly,postgres-16,pgoutput-decoder" ]
      fail-fast: false
    name: "PostgreSQL - ${{ matrix.profile }}"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.postgresql-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-postgres
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
          profile: ${{ matrix.profile }}

  # Approx 1h 45m
  build_sqlserver:
    name: "SQL Server"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.sqlserver-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-sqlserver
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  # Approx 6m
  build_oracle:
    name: "Oracle"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.oracle-changed == 'true' || needs.file_changes.outputs.oracle-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-oracle
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  # Approx 2m
  build_outbox:
    name: "Quarkus Outbox Extension"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.outbox-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-outbox
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  # Approx 5m
  build_rest_extension:
    name: "REST Extension"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.rest-extension-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-rest-extension
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  # Approx 2m
  build_schema_generator:
    name: "Schema Generator"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-schema-generator
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  # Approx 5m
  build_debezium_testing:
    name: "Testing Module"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.debezium-testing-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-testing
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  # Approx 3m
  build_storage:
    name: "Storage Module"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.storage-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-storage
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  # Approx 25m
  build_cassandra:
    name: "Cassandra"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
    steps:
      - name: Checkout Action (Debezium Core)
        uses: actions/checkout@v4
        with:
          path: core

      - name: Checkout Action (Cassandra)
        uses: actions/checkout@v4
        with:
          repository: debezium/debezium-connector-cassandra
          path: cassandra
          ref: ${{ github.event.pull_request.base.ref }} 

      - uses: ./core/.github/actions/setup-java

      - uses: ./core/.github/actions/maven-cache
        with:
          # We specifically only use the hash of the pom files from the core repository
          key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}

      - uses: ./core/.github/actions/build-debezium-cassandra
        with:
          path-core: core
          path-cassandra: cassandra

  # Approx 1h
  build_db2:
    name: "Db2"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
    steps:
      - name: Checkout Action (Debezium Core)
        uses: actions/checkout@v4
        with:
          path: core

      - name: Checkout Action (Db2)
        uses: actions/checkout@v4
        with:
          repository: debezium/debezium-connector-db2
          path: db2
          ref: ${{ github.event.pull_request.base.ref }} 

      - uses: ./core/.github/actions/setup-java

      - uses: ./core/.github/actions/maven-cache
        with:
          # We specifically only use the hash of the pom files from the core repository
          key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}

      - uses: ./core/.github/actions/build-debezium-db2
        with:
          path-core: core
          path-db2: db2

  build_ibmi:
    name: "IBMi"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
    steps:
      - name: Checkout Action (Debezium Core)
        uses: actions/checkout@v4
        with:
          path: core

      - name: Checkout Action (IBMi)
        uses: actions/checkout@v4
        with:
          repository: debezium/debezium-connector-ibmi
          path: ibmi
          ref: ${{ github.event.pull_request.base.ref }} 

      - uses: ./core/.github/actions/setup-java

      - uses: ./core/.github/actions/maven-cache
        with:
          # We specifically only use the hash of the pom files from the core repository
          key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}

      - uses: ./core/.github/actions/build-debezium-ibmi
        with:
          path-core: core
          path-ibmi: ibmi

  # Approx 45m
  build_informix:
    name: "Informix"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
    steps:
      - name: Checkout Action (Debezium Core)
        uses: actions/checkout@v4
        with:
          path: core

      - name: Checkout Action (Informix)
        uses: actions/checkout@v4
        with:
          repository: debezium/debezium-connector-informix
          path: informix
          ref: ${{ github.event.pull_request.base.ref }} 

      - uses: ./core/.github/actions/setup-java

      - uses: ./core/.github/actions/maven-cache
        with:
          # We specifically only use the hash of the pom files from the core repository
          key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}

      - uses: ./core/.github/actions/build-debezium-informix
        with:
          path-core: core
          path-informix: informix

  # Approx 20m
  build_vitess:
    name: "Vitess"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
    steps:
      - name: Checkout Action (Debezium Core)
        uses: actions/checkout@v4
        with:
          path: core

      - name: Checkout Action (Vitess)
        uses: actions/checkout@v4
        with:
          repository: debezium/debezium-connector-vitess
          path: vitess
          ref: ${{ github.event.pull_request.base.ref }} 

      - uses: ./core/.github/actions/setup-java

      - uses: ./core/.github/actions/maven-cache
        with:
          # We specifically only use the hash of the pom files from the core repository
          key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}

      - uses: ./core/.github/actions/build-debezium-vitess
        with:
          path-core: core
          path-vitess: vitess

  # Approx 7m
  build_spanner:
    name: "Spanner"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
    steps:
      - name: Checkout Action (Debezium Core)
        uses: actions/checkout@v4
        with:
          path: core

      - name: Checkout Action (Spanner)
        uses: actions/checkout@v4
        with:
          repository: debezium/debezium-connector-spanner
          path: spanner
          ref: ${{ github.event.pull_request.base.ref }} 

      - uses: ./core/.github/actions/setup-java

      - uses: ./core/.github/actions/maven-cache
        with:
          # We specifically only use the hash of the pom files from the core repository
          key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}

      - uses: ./core/.github/actions/build-debezium-spanner
        with:
          path-core: core
          path-spanner: spanner

  # Approx 1m
  build_jdbc:
    name: "JDBC"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.mariadb-changed == 'true' || needs.file_changes.outputs.postgresql-changed == 'true' || needs.file_changes.outputs.sqlserver-changed == 'true' || needs.file_changes.outputs.sqlserver-changed == 'true' || needs.file_changes.outputs.db2-changed == 'true' }}
    steps:
      - name: Checkout Action
        uses: actions/checkout@v4

      - uses: ./.github/actions/build-debezium-jdbc
        with:
          maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}

  build_debezium_server:
    name: "Debezium Server"
    needs: [ check_style, file_changes ]
    runs-on: ubuntu-latest
    if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
    steps:
      - name: Checkout Action (Debezium Core)
        uses: actions/checkout@v4
        with:
          path: core

      - name: Checkout Action (Debezium Server)
        uses: actions/checkout@v4
        with:
          repository: debezium/debezium-server
          path: server
          ref: ${{ github.event.pull_request.base.ref }} 

      - uses: ./core/.github/actions/setup-java

      - uses: ./core/.github/actions/maven-cache
        with:
          # We specifically only use the hash of the pom files from the core repository
          key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}

      - uses: ./core/.github/actions/build-debezium-server
        with:
          path-core: core
          path-server: server