From 6e76e119658bfca8448ecf0d455982d3a6d73693 Mon Sep 17 00:00:00 2001 From: Artem Sidorkin Date: Tue, 21 Apr 2020 19:17:40 +0300 Subject: [PATCH 1/2] Removed kafka partitions property. Updated docker images. --- _set-env.sh | 1 - docker-compose-mysql-binlog.yml | 108 +++++++++-------- docker-compose-mysql-polling.yml | 113 +++++++++--------- docker-compose-postgres-polling.yml | 110 +++++++++-------- docker-compose-postgres-wal.yml | 102 ++++++++-------- .../e2e/tests/TestExportSnapshots.java | 5 - .../rebuild/DomainSnapshotExportService.java | 7 +- .../rebuild/SnapshotConfiguration.java | 3 +- .../SnapshotterConfigurationProperties.java | 7 -- .../IterationOverDomainEntitiesTest.java | 3 +- 10 files changed, 232 insertions(+), 227 deletions(-) diff --git a/_set-env.sh b/_set-env.sh index a224a36..f4f8bfe 100644 --- a/_set-env.sh +++ b/_set-env.sh @@ -12,7 +12,6 @@ echo DOCKER_HOST_IP is $DOCKER_HOST_IP export EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS=$DOCKER_HOST_IP:9092 -export EVENTUATE_KAFKA_PARTITIONS=2 export EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING=$DOCKER_HOST_IP:2181 export CDC_SERVICE_URL=http://$DOCKER_HOST_IP:8099 export SPRING_DATA_MONGODB_URI=mongodb://${DOCKER_HOST_IP?}/export_test diff --git a/docker-compose-mysql-binlog.yml b/docker-compose-mysql-binlog.yml index 7836fb5..b143635 100755 --- a/docker-compose-mysql-binlog.yml +++ b/docker-compose-mysql-binlog.yml @@ -1,56 +1,60 @@ -zookeeper: - image: eventuateio/eventuateio-local-zookeeper:0.30.0.M14 - ports: - - 2181:2181 - - 2888:2888 - - 3888:3888 +version: '3' +services: + zookeeper: + image: confluentinc/cp-zookeeper:5.2.4 + ports: + - 2181:2181 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 -kafka: - image: eventuateio/eventuateio-local-kafka:0.30.0.M14 - ports: - - 9092:9092 - links: - - zookeeper - environment: - - ADVERTISED_HOST_NAME=${DOCKER_HOST_IP} - - KAFKA_HEAP_OPTS=-Xmx320m -Xms320m - - ZOOKEEPER_SERVERS=zookeeper:2181 + kafka: + image: "confluentinc/cp-enterprise-kafka:5.2.4" + ports: + - 9092:9092 + depends_on: + - zookeeper + environment: + KAFKA_LISTENERS: LC://kafka:29092,LX://kafka:9092 + KAFKA_ADVERTISED_LISTENERS: LC://kafka:29092,LX://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LC:PLAINTEXT,LX:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LC + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + mysql: + image: eventuateio/eventuate-mysql:0.9.0.RC5 + ports: + - 3306:3306 + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_USER=mysqluser + - MYSQL_PASSWORD=mysqlpw -mysql: - image: eventuateio/eventuate-tram-mysql:0.20.0.M10 - ports: - - 3306:3306 - environment: - - MYSQL_ROOT_PASSWORD=rootpassword - - MYSQL_USER=mysqluser - - MYSQL_PASSWORD=mysqlpw + mongodb: + image: mongo:3.0.4 + hostname: mongodb + command: mongod --smallfiles + ports: + - "27017:27017" -mongodb: - image: mongo:3.0.4 - hostname: mongodb - command: mongod --smallfiles - ports: - - "27017:27017" - -cdcservice: - image: eventuateio/eventuate-cdc-service:local-test - ports: - - "8099:8080" - links: - - mysql - - kafka - - zookeeper - environment: - SPRING_DATASOURCE_URL: jdbc:mysql://mysql/eventuate - SPRING_DATASOURCE_USERNAME: mysqluser - SPRING_DATASOURCE_PASSWORD: mysqlpw - SPRING_DATASOURCE_DRIVER_CLASS_NAME: com.mysql.jdbc.Driver - EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:29092 - EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 - EVENTUATELOCAL_CDC_DB_USER_NAME: root - EVENTUATELOCAL_CDC_DB_PASSWORD: rootpassword - EVENTUATELOCAL_CDC_READER_NAME: Reader - EVENTUATELOCAL_CDC_OFFSET_STORE_KEY: MySqlBinlog - EVENTUATELOCAL_CDC_MYSQL_BINLOG_CLIENT_UNIQUE_ID: 1234567890 - EVENTUATELOCAL_CDC_READ_OLD_DEBEZIUM_DB_OFFSET_STORAGE_TOPIC: "false" + cdcservice: + image: eventuateio/eventuate-cdc-service:0.6.0.RC3 + ports: + - "8099:8080" + depends_on: + - mysql + - kafka + - zookeeper + environment: + SPRING_DATASOURCE_URL: jdbc:mysql://mysql/eventuate + SPRING_DATASOURCE_USERNAME: mysqluser + SPRING_DATASOURCE_PASSWORD: mysqlpw + SPRING_DATASOURCE_DRIVER_CLASS_NAME: com.mysql.jdbc.Driver + EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:29092 + EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 + EVENTUATELOCAL_CDC_DB_USER_NAME: root + EVENTUATELOCAL_CDC_DB_PASSWORD: rootpassword + EVENTUATELOCAL_CDC_READER_NAME: Reader + EVENTUATELOCAL_CDC_OFFSET_STORE_KEY: MySqlBinlog + EVENTUATELOCAL_CDC_MYSQL_BINLOG_CLIENT_UNIQUE_ID: 1234567890 + EVENTUATELOCAL_CDC_READ_OLD_DEBEZIUM_DB_OFFSET_STORAGE_TOPIC: "false" diff --git a/docker-compose-mysql-polling.yml b/docker-compose-mysql-polling.yml index 6b7a2b5..cf34194 100755 --- a/docker-compose-mysql-polling.yml +++ b/docker-compose-mysql-polling.yml @@ -1,58 +1,63 @@ -zookeeper: - image: eventuateio/eventuateio-local-zookeeper:0.30.0.M14 - ports: - - 2181:2181 - - 2888:2888 - - 3888:3888 +version: '3' +services: + zookeeper: + image: confluentinc/cp-zookeeper:5.2.4 + ports: + - 2181:2181 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 -kafka: - image: eventuateio/eventuateio-local-kafka:0.30.0.M14 - ports: - - 9092:9092 - links: - - zookeeper - environment: - - ADVERTISED_HOST_NAME=${DOCKER_HOST_IP} - - KAFKA_HEAP_OPTS=-Xmx320m -Xms320m - - ZOOKEEPER_SERVERS=zookeeper:2181 + kafka: + image: "confluentinc/cp-enterprise-kafka:5.2.4" + ports: + - 9092:9092 + - 29092:29092 + depends_on: + - zookeeper + environment: + KAFKA_LISTENERS: LC://kafka:29092,LX://kafka:9092 + KAFKA_ADVERTISED_LISTENERS: LC://kafka:29092,LX://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LC:PLAINTEXT,LX:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LC + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + mysql: + image: eventuateio/eventuate-mysql:0.9.0.RC5 + ports: + - 3306:3306 + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_USER=mysqluser + - MYSQL_PASSWORD=mysqlpw -mysql: - image: eventuateio/eventuate-tram-mysql:0.20.0.M10 - ports: - - 3306:3306 - environment: - - MYSQL_ROOT_PASSWORD=rootpassword - - MYSQL_USER=mysqluser - - MYSQL_PASSWORD=mysqlpw + mongodb: + image: mongo:3.0.4 + hostname: mongodb + command: mongod --smallfiles + ports: + - "27017:27017" -mongodb: - image: mongo:3.0.4 - hostname: mongodb - command: mongod --smallfiles - ports: - - "27017:27017" - -cdcservice: - image: eventuateio/eventuate-cdc-service:local-test - ports: - - "8099:8080" - links: - - mysql - - kafka - - zookeeper - environment: - SPRING_DATASOURCE_URL: jdbc:mysql://mysql/eventuate - SPRING_DATASOURCE_USERNAME: mysqluser - SPRING_DATASOURCE_PASSWORD: mysqlpw - SPRING_DATASOURCE_DRIVER_CLASS_NAME: com.mysql.jdbc.Driver - EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 - EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 - EVENTUATELOCAL_CDC_DB_USER_NAME: root - EVENTUATELOCAL_CDC_DB_PASSWORD: rootpassword - EVENTUATELOCAL_CDC_POLLING_INTERVAL_IN_MILLISECONDS: 500 - EVENTUATELOCAL_CDC_MAX_EVENTS_PER_POLLING: 1000 - EVENTUATELOCAL_CDC_MAX_ATTEMPTS_FOR_POLLING: 100 - EVENTUATELOCAL_CDC_POLLING_RETRY_INTERVAL_IN_MILLISECONDS: 500 - EVENTUATELOCAL_CDC_READER_NAME: Reader - SPRING_PROFILES_ACTIVE: EventuatePolling \ No newline at end of file + cdcservice: + image: eventuateio/eventuate-cdc-service:0.6.0.RC3 + ports: + - "8099:8080" + depends_on: + - mysql + - kafka + - zookeeper + environment: + SPRING_DATASOURCE_URL: jdbc:mysql://mysql/eventuate + SPRING_DATASOURCE_USERNAME: mysqluser + SPRING_DATASOURCE_PASSWORD: mysqlpw + SPRING_DATASOURCE_DRIVER_CLASS_NAME: com.mysql.jdbc.Driver + EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 + EVENTUATELOCAL_CDC_DB_USER_NAME: root + EVENTUATELOCAL_CDC_DB_PASSWORD: rootpassword + EVENTUATELOCAL_CDC_POLLING_INTERVAL_IN_MILLISECONDS: 500 + EVENTUATELOCAL_CDC_MAX_EVENTS_PER_POLLING: 1000 + EVENTUATELOCAL_CDC_MAX_ATTEMPTS_FOR_POLLING: 100 + EVENTUATELOCAL_CDC_POLLING_RETRY_INTERVAL_IN_MILLISECONDS: 500 + EVENTUATELOCAL_CDC_READER_NAME: Reader + SPRING_PROFILES_ACTIVE: EventuatePolling \ No newline at end of file diff --git a/docker-compose-postgres-polling.yml b/docker-compose-postgres-polling.yml index 5f49175..0b668ac 100755 --- a/docker-compose-postgres-polling.yml +++ b/docker-compose-postgres-polling.yml @@ -1,56 +1,62 @@ -zookeeper: - image: eventuateio/eventuateio-local-zookeeper:0.30.0.M14 - ports: - - 2181:2181 - - 2888:2888 - - 3888:3888 +version: '3' +services: + zookeeper: + image: confluentinc/cp-zookeeper:5.2.4 + ports: + - 2181:2181 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 -kafka: - image: eventuateio/eventuateio-local-kafka:0.30.0.M14 - ports: - - 9092:9092 - links: - - zookeeper - environment: - - ADVERTISED_HOST_NAME=${DOCKER_HOST_IP} - - KAFKA_HEAP_OPTS=-Xmx320m -Xms320m - - ZOOKEEPER_SERVERS=zookeeper:2181 + kafka: + image: "confluentinc/cp-enterprise-kafka:5.2.4" + ports: + - 9092:9092 + - 29092:29092 + depends_on: + - zookeeper + environment: + KAFKA_LISTENERS: LC://kafka:29092,LX://kafka:9092 + KAFKA_ADVERTISED_LISTENERS: LC://kafka:29092,LX://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LC:PLAINTEXT,LX:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LC + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 -postgres: - image: eventuateio/eventuate-tram-postgres:0.20.0.M10 - ports: - - 5432:5432 - environment: - POSTGRES_USER: eventuate - POSTGRES_PASSWORD: eventuate + postgres: + image: eventuateio/eventuate-postgres:0.9.0.RC5 + ports: + - 5432:5432 + environment: + POSTGRES_USER: eventuate + POSTGRES_PASSWORD: eventuate -mongodb: - image: mongo:3.0.4 - hostname: mongodb - command: mongod --smallfiles - ports: - - "27017:27017" + mongodb: + image: mongo:3.0.4 + hostname: mongodb + command: mongod --smallfiles + ports: + - "27017:27017" -cdcservice: - image: eventuateio/eventuate-cdc-service:local-test - links: - - postgres - - kafka - - zookeeper - ports: - - "8099:8080" - environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres/eventuate - SPRING_DATASOURCE_USERNAME: eventuate - SPRING_DATASOURCE_PASSWORD: eventuate - SPRING_DATASOURCE_TEST_ON_BORROW: "true" - SPRING_DATASOURCE_VALIDATION_QUERY: SELECT 1 - SPRING_DATASOURCE_DRIVER_CLASS_NAME: org.postgresql.Driver - EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 - EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 - EVENTUATELOCAL_CDC_POLLING_INTERVAL_IN_MILLISECONDS: 500 - EVENTUATELOCAL_CDC_MAX_EVENTS_PER_POLLING: 1000 - EVENTUATELOCAL_CDC_MAX_ATTEMPTS_FOR_POLLING: 100 - EVENTUATELOCAL_CDC_POLLING_RETRY_INTERVAL_IN_MILLISECONDS: 500 - EVENTUATELOCAL_CDC_READER_NAME: Reader - SPRING_PROFILES_ACTIVE: EventuatePolling \ No newline at end of file + cdcservice: + image: eventuateio/eventuate-cdc-service:0.6.0.RC3 + depends_on: + - postgres + - kafka + - zookeeper + ports: + - "8099:8080" + environment: + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres/eventuate + SPRING_DATASOURCE_USERNAME: eventuate + SPRING_DATASOURCE_PASSWORD: eventuate + SPRING_DATASOURCE_TEST_ON_BORROW: "true" + SPRING_DATASOURCE_VALIDATION_QUERY: SELECT 1 + SPRING_DATASOURCE_DRIVER_CLASS_NAME: org.postgresql.Driver + EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 + EVENTUATELOCAL_CDC_POLLING_INTERVAL_IN_MILLISECONDS: 500 + EVENTUATELOCAL_CDC_MAX_EVENTS_PER_POLLING: 1000 + EVENTUATELOCAL_CDC_MAX_ATTEMPTS_FOR_POLLING: 100 + EVENTUATELOCAL_CDC_POLLING_RETRY_INTERVAL_IN_MILLISECONDS: 500 + EVENTUATELOCAL_CDC_READER_NAME: Reader + SPRING_PROFILES_ACTIVE: EventuatePolling \ No newline at end of file diff --git a/docker-compose-postgres-wal.yml b/docker-compose-postgres-wal.yml index 1f89016..f9b6cc4 100755 --- a/docker-compose-postgres-wal.yml +++ b/docker-compose-postgres-wal.yml @@ -1,52 +1,58 @@ -zookeeper: - image: eventuateio/eventuateio-local-zookeeper:0.30.0.M14 - ports: - - 2181:2181 - - 2888:2888 - - 3888:3888 +version: '3' +services: + zookeeper: + image: confluentinc/cp-zookeeper:5.2.4 + ports: + - 2181:2181 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 -kafka: - image: eventuateio/eventuateio-local-kafka:0.30.0.M14 - ports: - - 9092:9092 - links: - - zookeeper - environment: - - ADVERTISED_HOST_NAME=${DOCKER_HOST_IP} - - KAFKA_HEAP_OPTS=-Xmx320m -Xms320m - - ZOOKEEPER_SERVERS=zookeeper:2181 + kafka: + image: "confluentinc/cp-enterprise-kafka:5.2.4" + ports: + - 9092:9092 + - 29092:29092 + depends_on: + - zookeeper + environment: + KAFKA_LISTENERS: LC://kafka:29092,LX://kafka:9092 + KAFKA_ADVERTISED_LISTENERS: LC://kafka:29092,LX://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LC:PLAINTEXT,LX:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LC + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 -postgres: - image: eventuateio/eventuate-tram-postgres:0.20.0.M10 - ports: - - 5432:5432 - environment: - POSTGRES_USER: eventuate - POSTGRES_PASSWORD: eventuate + postgres: + image: eventuateio/eventuate-postgres:0.9.0.RC5 + ports: + - 5432:5432 + environment: + POSTGRES_USER: eventuate + POSTGRES_PASSWORD: eventuate -mongodb: - image: mongo:3.0.4 - hostname: mongodb - command: mongod --smallfiles - ports: - - "27017:27017" + mongodb: + image: mongo:3.0.4 + hostname: mongodb + command: mongod --smallfiles + ports: + - "27017:27017" -cdcservice: - image: eventuateio/eventuate-cdc-service:local-test - links: - - postgres - - kafka - - zookeeper - ports: - - "8099:8080" - environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres/eventuate - SPRING_DATASOURCE_USERNAME: eventuate - SPRING_DATASOURCE_PASSWORD: eventuate - SPRING_DATASOURCE_TEST_ON_BORROW: "true" - SPRING_DATASOURCE_VALIDATION_QUERY: SELECT 1 - SPRING_DATASOURCE_DRIVER_CLASS_NAME: org.postgresql.Driver - EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 - EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 - EVENTUATELOCAL_CDC_READER_NAME: Reader - SPRING_PROFILES_ACTIVE: PostgresWal \ No newline at end of file + cdcservice: + image: eventuateio/eventuate-cdc-service:0.6.0.RC3 + depends_on: + - postgres + - kafka + - zookeeper + ports: + - "8099:8080" + environment: + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres/eventuate + SPRING_DATASOURCE_USERNAME: eventuate + SPRING_DATASOURCE_PASSWORD: eventuate + SPRING_DATASOURCE_TEST_ON_BORROW: "true" + SPRING_DATASOURCE_VALIDATION_QUERY: SELECT 1 + SPRING_DATASOURCE_DRIVER_CLASS_NAME: org.postgresql.Driver + EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 + EVENTUATELOCAL_CDC_READER_NAME: Reader + SPRING_PROFILES_ACTIVE: PostgresWal \ No newline at end of file diff --git a/eventuate-tram-view-support-e2e-tests/src/test/java/io/eventuate/tram/viewsupport/e2e/tests/TestExportSnapshots.java b/eventuate-tram-view-support-e2e-tests/src/test/java/io/eventuate/tram/viewsupport/e2e/tests/TestExportSnapshots.java index 2114d5c..cddeac9 100644 --- a/eventuate-tram-view-support-e2e-tests/src/test/java/io/eventuate/tram/viewsupport/e2e/tests/TestExportSnapshots.java +++ b/eventuate-tram-view-support-e2e-tests/src/test/java/io/eventuate/tram/viewsupport/e2e/tests/TestExportSnapshots.java @@ -33,9 +33,6 @@ public class TestExportSnapshots { @Autowired private DomainSnapshotExportService domainEntityDomainSnapshotExportService; - @Autowired - private SnapshotterConfigurationProperties snapshotterConfigurationProperties; - @LocalServerPort private int port; @@ -59,8 +56,6 @@ public void testThatViewsMatchOriginalEntitiesAfterExport() { TopicPartitionOffset[] topicPartitionOffsets = restTemplate.postForObject(String.format("http://localhost:%s/export/test-domain-entity", port), null, TopicPartitionOffset[].class); - Assert.assertEquals(snapshotterConfigurationProperties.getKafkaPartitions(), topicPartitionOffsets.length); - for (TopicPartitionOffset topicPartitionOffset : topicPartitionOffsets) { Assert.assertTrue(topicPartitionOffset.getOffset() > 0); Assert.assertEquals(TestDomainEntity.class.getName(), topicPartitionOffset.getTopic()); diff --git a/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/DomainSnapshotExportService.java b/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/DomainSnapshotExportService.java index b73a9e0..87aa52c 100644 --- a/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/DomainSnapshotExportService.java +++ b/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/DomainSnapshotExportService.java @@ -36,7 +36,6 @@ public class DomainSnapshotExportService { private String readerName; private int maxIterationsToCheckCdcProcessing; private int timeoutBetweenCdcProcessingCheckingIterationsInMilliseconds; - private int kafkaPartitions; private RestTemplate restTemplate = new RestTemplate(); public DomainSnapshotExportService(EventuateKafkaProducer eventuateKafkaProducer, @@ -51,8 +50,7 @@ public DomainSnapshotExportService(EventuateKafkaProducer eventuateKafkaProducer String cdcStatusServiceEndPoint, String readerName, int maxIterationsToCheckCdcProcessing, - int timeoutBetweenCdcProcessingCheckingIterationsInMilliseconds, - int kafkaPartitions) { + int timeoutBetweenCdcProcessingCheckingIterationsInMilliseconds) { this.eventuateKafkaProducer = eventuateKafkaProducer; this.dbLockService = dbLockService; @@ -67,7 +65,6 @@ public DomainSnapshotExportService(EventuateKafkaProducer eventuateKafkaProducer this.readerName = readerName; this.maxIterationsToCheckCdcProcessing = maxIterationsToCheckCdcProcessing; this.timeoutBetweenCdcProcessingCheckingIterationsInMilliseconds = timeoutBetweenCdcProcessingCheckingIterationsInMilliseconds; - this.kafkaPartitions = kafkaPartitions; } public List exportSnapshots() { @@ -127,6 +124,8 @@ private RecordMetadata getRecordMetadataFromFuture(CompletableFuture completa private List publishSnapshotEvents() { List> metadata = new ArrayList<>(); + int kafkaPartitions = eventuateKafkaProducer.partitionsFor(domainClass.getName()).size(); + for (int i = 0; i < kafkaPartitions; i++) { Message message = MessageBuilder .withPayload("") diff --git a/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/SnapshotConfiguration.java b/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/SnapshotConfiguration.java index 2ad2ab6..13542b0 100644 --- a/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/SnapshotConfiguration.java +++ b/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/SnapshotConfiguration.java @@ -67,7 +67,6 @@ public DomainSnapshotExportServiceFactory domainSnapshotExportServiceFact snapshotterConfigurationProperties.getCdcStatusServiceEndPoint(), readerName, snapshotterConfigurationProperties.getMaxIterationsToCheckCdcProcessing(), - snapshotterConfigurationProperties.getTimeoutBetweenCdcProcessingCheckingIterationsInMilliseconds(), - snapshotterConfigurationProperties.getKafkaPartitions()); + snapshotterConfigurationProperties.getTimeoutBetweenCdcProcessingCheckingIterationsInMilliseconds()); } } diff --git a/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/SnapshotterConfigurationProperties.java b/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/SnapshotterConfigurationProperties.java index e052eb4..d25bf38 100644 --- a/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/SnapshotterConfigurationProperties.java +++ b/eventuate-tram-view-support-rebuild/src/main/java/io/eventuate/tram/viewsupport/rebuild/SnapshotterConfigurationProperties.java @@ -18,9 +18,6 @@ public class SnapshotterConfigurationProperties { @Value("${timeout.between.cdc.processing.checking.iterations.in.milliseconds:#{1000}}") private int timeoutBetweenCdcProcessingCheckingIterationsInMilliseconds; - @Value("${eventuate.kafka.partitions}") - private int kafkaPartitions; - public int getDomainRepositoryPageSize() { return domainRepositoryPageSize; } @@ -40,8 +37,4 @@ public int getMaxIterationsToCheckCdcProcessing() { public int getTimeoutBetweenCdcProcessingCheckingIterationsInMilliseconds() { return timeoutBetweenCdcProcessingCheckingIterationsInMilliseconds; } - - public int getKafkaPartitions() { - return kafkaPartitions; - } } diff --git a/eventuate-tram-view-support-rebuild/src/test/java/io/eventuate/tram/viewsupport/rebuild/IterationOverDomainEntitiesTest.java b/eventuate-tram-view-support-rebuild/src/test/java/io/eventuate/tram/viewsupport/rebuild/IterationOverDomainEntitiesTest.java index b2756f6..c8fbeac 100644 --- a/eventuate-tram-view-support-rebuild/src/test/java/io/eventuate/tram/viewsupport/rebuild/IterationOverDomainEntitiesTest.java +++ b/eventuate-tram-view-support-rebuild/src/test/java/io/eventuate/tram/viewsupport/rebuild/IterationOverDomainEntitiesTest.java @@ -70,8 +70,7 @@ public void testIteration(int elements) { snapshotterConfigurationProperties.getCdcStatusServiceEndPoint(), null, snapshotterConfigurationProperties.getMaxIterationsToCheckCdcProcessing(), - snapshotterConfigurationProperties.getTimeoutBetweenCdcProcessingCheckingIterationsInMilliseconds(), - 2); + snapshotterConfigurationProperties.getTimeoutBetweenCdcProcessingCheckingIterationsInMilliseconds()); testRepository.deleteAll(); From 9597e1d97ce796b7754d22e70efb31ae42d41ef5 Mon Sep 17 00:00:00 2001 From: Artem Sidorkin Date: Tue, 21 Apr 2020 19:33:58 +0300 Subject: [PATCH 2/2] Updated libraries to released versions. --- docker-compose-mysql-binlog.yml | 4 ++-- docker-compose-mysql-polling.yml | 4 ++-- docker-compose-postgres-polling.yml | 4 ++-- docker-compose-postgres-wal.yml | 4 ++-- gradle.properties | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose-mysql-binlog.yml b/docker-compose-mysql-binlog.yml index b143635..bd42082 100755 --- a/docker-compose-mysql-binlog.yml +++ b/docker-compose-mysql-binlog.yml @@ -22,7 +22,7 @@ services: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 mysql: - image: eventuateio/eventuate-mysql:0.9.0.RC5 + image: eventuateio/eventuate-mysql:0.9.0.RELEASE ports: - 3306:3306 environment: @@ -38,7 +38,7 @@ services: - "27017:27017" cdcservice: - image: eventuateio/eventuate-cdc-service:0.6.0.RC3 + image: eventuateio/eventuate-cdc-service:0.6.0.RELEASE ports: - "8099:8080" depends_on: diff --git a/docker-compose-mysql-polling.yml b/docker-compose-mysql-polling.yml index cf34194..a5f8e96 100755 --- a/docker-compose-mysql-polling.yml +++ b/docker-compose-mysql-polling.yml @@ -23,7 +23,7 @@ services: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 mysql: - image: eventuateio/eventuate-mysql:0.9.0.RC5 + image: eventuateio/eventuate-mysql:0.9.0.RELEASE ports: - 3306:3306 environment: @@ -39,7 +39,7 @@ services: - "27017:27017" cdcservice: - image: eventuateio/eventuate-cdc-service:0.6.0.RC3 + image: eventuateio/eventuate-cdc-service:0.6.0.RELEASE ports: - "8099:8080" depends_on: diff --git a/docker-compose-postgres-polling.yml b/docker-compose-postgres-polling.yml index 0b668ac..65418b4 100755 --- a/docker-compose-postgres-polling.yml +++ b/docker-compose-postgres-polling.yml @@ -23,7 +23,7 @@ services: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 postgres: - image: eventuateio/eventuate-postgres:0.9.0.RC5 + image: eventuateio/eventuate-postgres:0.9.0.RELEASE ports: - 5432:5432 environment: @@ -38,7 +38,7 @@ services: - "27017:27017" cdcservice: - image: eventuateio/eventuate-cdc-service:0.6.0.RC3 + image: eventuateio/eventuate-cdc-service:0.6.0.RELEASE depends_on: - postgres - kafka diff --git a/docker-compose-postgres-wal.yml b/docker-compose-postgres-wal.yml index f9b6cc4..36d0d9b 100755 --- a/docker-compose-postgres-wal.yml +++ b/docker-compose-postgres-wal.yml @@ -23,7 +23,7 @@ services: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 postgres: - image: eventuateio/eventuate-postgres:0.9.0.RC5 + image: eventuateio/eventuate-postgres:0.9.0.RELEASE ports: - 5432:5432 environment: @@ -38,7 +38,7 @@ services: - "27017:27017" cdcservice: - image: eventuateio/eventuate-cdc-service:0.6.0.RC3 + image: eventuateio/eventuate-cdc-service:0.6.0.RELEASE depends_on: - postgres - kafka diff --git a/gradle.properties b/gradle.properties index 753b8a4..c5a43aa 100644 --- a/gradle.properties +++ b/gradle.properties @@ -7,6 +7,6 @@ eventuateMavenRepoUrl=https://dl.bintray.com/eventuateio-oss/eventuate-maven-rel springBootVersion=2.1.1.RELEASE eventuateClientVersion=0.23.0.RELEASE eventuateTramVersion=0.24.0.RELEASE -eventuateUtilVersion=0.3.0.RELEASE +eventuateUtilVersion=0.4.0.RELEASE eventuateMessagingKafkaVersion=0.9.0.RELEASE version=0.1.0-SNAPSHOT \ No newline at end of file