Skip to content

CXP-2956: Implement keyset pagination in debezium sql server connector (2nd iteration) #23

CXP-2956: Implement keyset pagination in debezium sql server connector (2nd iteration)

CXP-2956: Implement keyset pagination in debezium sql server connector (2nd iteration) #23

name: Build Debezium (PR)
on:
pull_request:
branches:
- main
- 1.*
- 2.*
env:
MAVEN_FULL_BUILD_PROJECTS: "\\!debezium-microbenchmark-oracle"
# Pushes to each pull request will trigger a cancellation of any existing push jobs for that pull request,
# and will restart the build based on the latest push data for that specific pull request.
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}
cancel-in-progress: true
jobs:
# This job is responsible for inspecting the changes in the repository and setting specific output variables
# that subsequent jobs can use to determine whether a particular job should be skipped or included.
file_changes:
name: "Detect repository changes"
runs-on: ubuntu-latest
outputs:
common-changed: ${{ steps.changed-files-common.outputs.any_changed }}
mongodb-changed: ${{ steps.changed-files-mongodb.outputs.any_changed }}
mysql-changed: ${{ steps.changed-files-mysql.outputs.any_changed }}
postgresql-changed: ${{ steps.changed-files-postgresql.outputs.any_changed }}
oracle-changed: ${{ steps.changed-files-oracle.outputs.any_changed }}
sqlserver-changed: ${{ steps.changed-files-sqlserver.outputs.any_changed }}
outbox-changed: ${{ steps.changed-files-outbox.outputs.any_changed }}
rest-extension-changed: ${{ steps.changed-files-rest-extension.outputs.any_changed }}
schema-generator-changed: ${{ steps.changed-files-schema-generator.outputs.any_changed }}
debezium-testing-changed: ${{ steps.changed-files-debezium-testing.outputs.any_changed }}
debezium-testing-mongodb-changed: ${{ steps.changed-files-debezium-testing-mongodb.outputs.any_changed }}
mysql-ddl-parser-changed: ${{ steps.changed-files-mysql-ddl-parser.outputs.any_changed }}
oracle-ddl-parser-changed: ${{ steps.changed-files-oracle-ddl-parser.outputs.any_changed }}
documentation-only-changed: ${{ steps.changed-files-documentation.outputs.only_changed }}
storage-only-changed: ${{ steps.changed-files-storage.outputs.only_changed }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get modified files (Common)
id: changed-files-common
uses: tj-actions/[email protected]
with:
files: |
support/checkstyle/**
support/revapi/**
debezium-api/**
debezium-assembly-descriptors/**
debezium-core/**
debezium-embedded/**
debezium-revapi/**
debezium-ide-configs/**
debezium-parent/pom.xml
debezium-bom/pom.xml
debezium-storage/**
debezium-scripting/**
pom.xml
.mvn/**
.github/actions/**
.github/workflows/debezium-workflow-pr.yml
- name: Get modified files (MongoDB)
id: changed-files-mongodb
uses: tj-actions/[email protected]
with:
files: |
debezium-connector-mongodb/**
- name: Get modified files (MySQL)
id: changed-files-mysql
uses: tj-actions/[email protected]
with:
files: |
debezium-connector-mysql/**
- name: Get modified files (PostgreSQL)
id: changed-files-postgresql
uses: tj-actions/[email protected]
with:
files: |
debezium-connector-postgres/**
- name: Get modified files (Oracle)
id: changed-files-oracle
uses: tj-actions/[email protected]
with:
files: |
debezium-connector-oracle/**
- name: Get modified files (SQL Server)
id: changed-files-sqlserver
uses: tj-actions/[email protected]
with:
files: |
debezium-connector-sqlserver/**
- name: Get modified files (Quarkus Outbox)
id: changed-files-outbox
uses: tj-actions/[email protected]
with:
files: |
debezium-quarkus-outbox/**
debezium-quarkus-outbox-common/**
debezium-quarkus-outbox-reactive/**
- name: Get modified files (REST Extension)
id: changed-files-rest-extension
uses: tj-actions/[email protected]
with:
files: |
debezium-connect-rest-extension/**
- name: Get modified files (Schema Generator)
id: changed-files-schema-generator
uses: tj-actions/[email protected]
with:
files: |
debezium-schema-generator/**
- name: Get modified files (Debezium Testing)
id: changed-files-debezium-testing
uses: tj-actions/[email protected]
with:
files: |
debezium-testing/**
- name: Get modified files (Debezium Testing MongoDB)
id: changed-files-debezium-testing-mongodb
uses: tj-actions/[email protected]
with:
files: |
debezium-testing/**/MongoDb*.java
- name: Get modified files (MySQL DDL parser)
id: changed-files-mysql-ddl-parser
uses: tj-actions/[email protected]
with:
files: |
debezium-ddl-parser/src/main/antlr4/io/debezium/ddl/parser/mysql/**
debezium-ddl-parser/src/main/java/io/debezium/antlr/**
debezium-ddl-parser/src/test/resources/mysql/examples/**
- name: Get modified files (Oracle DDL parser)
id: changed-files-oracle-ddl-parser
uses: tj-actions/[email protected]
with:
files: |
debezium-ddl-parser/src/main/antlr4/io/debezium/ddl/parser/oracle/**
debezium-ddl-parser/src/main/java/io/debezium/antlr/**
debezium-ddl-parser/src/main/java/io/debezium/ddl/parser/oracle/**
debezium-ddl-parser/src/test/resources/oracle/examples/**
- name: Get modified files (Documentation)
id: changed-files-documentation
uses: tj-actions/[email protected]
with:
files: |
documentation/**
- name: Get modified files (Storage)
id: changed-files-storage
uses: tj-actions/[email protected]
with:
files: |
debezium-storage/**
# Approx 1m
build_cache:
name: "Update Dependencies"
needs: [ file_changes ]
if: ${{ needs.file_changes.outputs.documentation-only-changed == 'false' }}
runs-on: ubuntu-latest
steps:
- name: Checkout Action (Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (Debezium Server)
uses: actions/checkout@v4
with:
repository: debezium/debezium-server
ref: bc0393bd6b6814e07f543292988620bcb3f9ca54
path: server
- uses: ./core/.github/actions/setup-java
# GitHub actions seem to struggle returning actions/cache cache-hit
# Directly use the cache action here to control whether to fetch dependencies
- id: maven-cache-check
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
restore-keys: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
# This step is responsible for pulling down artifacts
# Unfortunately due to the nature of how some of the maven workflows work, the only reliable way
# to guarantee fully seeding the maven cache is to run a full build. This step does not execute
# tests, formatting, checkstyle, nor import sorts.
#
# This check is conditioned so that if the cache-key was not found, we will execute this step.
# If the cache-key was found, this means we cannot update the cache and therefore we should
# not need to explicitly run this step. This should improve response of format/checkstyle
# errors to users faster.
#
# This job also explicitly excludes the "debezium-microbenchmark-oracle" module temporarily.
# There is a dependency on xstream.jar for this module that should be fixed and made to not
# be required so that the module can be built on GitHub Actions.
- name: Download dependencies (Core)
if: steps.maven-cache-check.outputs.cache-hit != 'true'
run: >
./core/mvnw -B -ntp clean install -f core/pom.xml
-pl ${{ env.MAVEN_FULL_BUILD_PROJECTS }}
-Dformat.skip=true
-Dcheckstyle.skip=true
-Dorg.slf4j.simpleLogger.showDateTime=true
-Dorg.slf4j.simpleLogger.dateTimeFormat="YYYY-MM-dd HH:mm:ss,SSS"
-DskipTests=true
-DskipITs=true
- name: Download dependencies (Debezium Server)
if: steps.maven-cache-check.outputs.cache-hit != 'true'
run: >
./server/mvnw -B -ntp clean install -f server/pom.xml
-Dformat.skip=true
-Dcheckstyle.skip=true
-Dorg.slf4j.simpleLogger.showDateTime=true
-Dorg.slf4j.simpleLogger.dateTimeFormat="YYYY-MM-dd HH:mm:ss,SSS"
-DskipTests=true
-DskipITs=true
# Approx 1m
check_style:
name: "Checkstyle and Formatting"
needs: [ build_cache ]
runs-on: ubuntu-latest
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/checkstyle-format
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
build_mongodb:
strategy:
# Runs each combination concurrently
matrix:
version-mongo-server: [ "5.0", "7.0" ]
fail-fast: false
name: "MongoDB ${{ matrix.version-mongo-server }}"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mongodb-changed == 'true' || needs.file_changes.outputs.debezium-testing-mongodb-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-mongodb
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
version-mongo-server: ${{ matrix.version-mongo-server }}
# Approx 40m each
build_mysql:
strategy:
# Runs each combination concurrently
matrix:
profile: [ "mysql-ci", "mysql-ci-gtids", "mysql-ci-percona", "mysql-ci-ssl" ]
version-mysql-server: [ "8.0", "8.2" ]
fail-fast: false
name: "MySQL ${{ matrix.version-mysql-server }} - ${{ matrix.profile }}"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-mysql
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
version-mysql-server: ${{ matrix.version-mysql-server }}
profile: ${{ matrix.profile }}
# Approx 40m each
build_mariadb:
strategy:
# Runs each combination concurrently
matrix:
profile: [ 'mysql-ci', 'mysql-ci-gtids' ]
fail-fast: false
name: "MariaDB - ${{ matrix.profile }}"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-mariadb
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
profile: ${{ matrix.profile }}
# Approx 40m each
build_postgresql:
strategy:
# Runs each combination concurrently
matrix:
profile: [ "assembly", "assembly,postgres-16,pgoutput-decoder" ]
fail-fast: false
name: "PostgreSQL - ${{ matrix.profile }}"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.postgresql-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-postgres
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
profile: ${{ matrix.profile }}
# Approx 1h 45m
build_sqlserver:
name: "SQL Server"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.sqlserver-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-sqlserver
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
# Approx 6m
build_oracle:
name: "Oracle"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.oracle-changed == 'true' || needs.file_changes.outputs.oracle-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-oracle
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
# Approx 2m
build_outbox:
name: "Quarkus Outbox Extension"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.outbox-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-outbox
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
# Approx 5m
build_rest_extension:
name: "REST Extension"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.rest-extension-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-rest-extension
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
# Approx 2m
build_schema_generator:
name: "Schema Generator"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-schema-generator
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
# Approx 5m
build_debezium_testing:
name: "Testing Module"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.debezium-testing-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-testing
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
# Approx 3m
build_storage:
name: "Storage Module"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.storage-changed == 'true' }}
steps:
- name: Checkout Action
uses: actions/checkout@v4
- uses: ./.github/actions/build-debezium-storage
with:
maven-cache-key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }}
# Approx 25m
build_cassandra:
name: "Cassandra"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
steps:
- name: Checkout Action (Debezium Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (Cassandra)
uses: actions/checkout@v4
with:
repository: debezium/debezium-connector-cassandra
path: cassandra
- uses: ./core/.github/actions/setup-java
- uses: ./core/.github/actions/maven-cache
with:
# We specifically only use the hash of the pom files from the core repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
- uses: ./core/.github/actions/build-debezium-cassandra
with:
path-core: core
path-cassandra: cassandra
# Approx 1h
build_db2:
name: "Db2"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
steps:
- name: Checkout Action (Debezium Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (Db2)
uses: actions/checkout@v4
with:
repository: debezium/debezium-connector-db2
path: db2
- uses: ./core/.github/actions/setup-java
- uses: ./core/.github/actions/maven-cache
with:
# We specifically only use the hash of the pom files from the core repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
- uses: ./core/.github/actions/build-debezium-db2
with:
path-core: core
path-db2: db2
build_ibmi:
name: "IBMi"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
steps:
- name: Checkout Action (Debezium Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (IBMi)
uses: actions/checkout@v4
with:
repository: debezium/debezium-connector-ibmi
path: ibmi
- uses: ./core/.github/actions/setup-java
- uses: ./core/.github/actions/maven-cache
with:
# We specifically only use the hash of the pom files from the core repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
- uses: ./core/.github/actions/build-debezium-ibmi
with:
path-core: core
path-ibmi: ibmi
# Approx 45m
build_informix:
name: "Informix"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
steps:
- name: Checkout Action (Debezium Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (Informix)
uses: actions/checkout@v4
with:
repository: debezium/debezium-connector-informix
path: informix
- uses: ./core/.github/actions/setup-java
- uses: ./core/.github/actions/maven-cache
with:
# We specifically only use the hash of the pom files from the core repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
- uses: ./core/.github/actions/build-debezium-informix
with:
path-core: core
path-informix: informix
# Approx 20m
build_vitess:
name: "Vitess"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
steps:
- name: Checkout Action (Debezium Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (Vitess)
uses: actions/checkout@v4
with:
repository: debezium/debezium-connector-vitess
path: vitess
- uses: ./core/.github/actions/setup-java
- uses: ./core/.github/actions/maven-cache
with:
# We specifically only use the hash of the pom files from the core repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
- uses: ./core/.github/actions/build-debezium-vitess
with:
path-core: core
path-vitess: vitess
# Approx 7m
build_spanner:
name: "Spanner"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
steps:
- name: Checkout Action (Debezium Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (Spanner)
uses: actions/checkout@v4
with:
repository: debezium/debezium-connector-spanner
path: spanner
- uses: ./core/.github/actions/setup-java
- uses: ./core/.github/actions/maven-cache
with:
# We specifically only use the hash of the pom files from the core repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
- uses: ./core/.github/actions/build-debezium-spanner
with:
path-core: core
path-spanner: spanner
# Approx 1m
build_jdbc:
name: "JDBC"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
steps:
- name: Checkout Action (Debezium Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (JDBC)
uses: actions/checkout@v4
with:
repository: debezium/debezium-connector-jdbc
path: jdbc
- uses: ./core/.github/actions/setup-java
- uses: ./core/.github/actions/maven-cache
with:
# We specifically only use the hash of the pom files from the core repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
- uses: ./core/.github/actions/build-debezium-jdbc
with:
path-core: core
path-jdbc: jdbc
build_debezium_server:
name: "Debezium Server"
needs: [ check_style, file_changes ]
runs-on: ubuntu-latest
if: ${{ needs.file_changes.outputs.common-changed == 'true' }}
steps:
- name: Checkout Action (Debezium Core)
uses: actions/checkout@v4
with:
path: core
- name: Checkout Action (Debezium Server)
uses: actions/checkout@v4
with:
repository: debezium/debezium-server
path: server
- uses: ./core/.github/actions/setup-java
- uses: ./core/.github/actions/maven-cache
with:
# We specifically only use the hash of the pom files from the core repository
key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }}
- uses: ./core/.github/actions/build-debezium-server
with:
path-core: core
path-server: server