From 73bc1397e4620d7c5052b65695a6c7a7cc508549 Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Wed, 31 Aug 2022 18:22:33 +0800 Subject: [PATCH 1/7] fixed null pointer exception #294 --- .../service/StateManagementServiceImpl.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/StateManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/StateManagementServiceImpl.java index 9c20f7eb2..0c20296e0 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/StateManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/StateManagementServiceImpl.java @@ -135,20 +135,28 @@ private synchronized void sendOnlineConfig() { /**connector status map*/ Map connectorStatusMap = connectorStatusStore.getKVMap(); connectorStatusMap.forEach((connectorName, connectorStatus) -> { + if (connectorStatus == null){ + return; + } // send status put(connectorStatus); }); /** task status map */ Map> taskStatusMap = taskStatusStore.getKVMap(); + if (taskStatusMap.isEmpty()){ + return; + } taskStatusMap.forEach((connectorName, taskStatusList) -> { + if (taskStatusList == null || taskStatusList.isEmpty()){ + return; + } taskStatusList.forEach(taskStatus -> { // send status put(taskStatus); }); }); } - /** * Stop dependent services (if needed) */ @@ -165,12 +173,21 @@ public void stop() { * pre persist */ private void prePersist() { - connAndTaskStatus.getConnectors().forEach((connectName, connectorStatus) -> { + Map> connectors = connAndTaskStatus.getConnectors(); + if (connectors.isEmpty()){ + return; + } + connectors.forEach((connectName, connectorStatus) -> { connectorStatusStore.put(connectName, connectorStatus.get()); Map> cacheTaskStatus = connAndTaskStatus.getTasks().row(connectName); + if (cacheTaskStatus == null){ + return; + } taskStatusStore.put(connectName, new ArrayList<>()); cacheTaskStatus.forEach((taskId, taskStatus) -> { - taskStatusStore.get(connectName).add(taskStatus.get()); + if (taskStatus != null){ + taskStatusStore.get(connectName).add(taskStatus.get()); + } }); }); } From 39b8eaeb3d9912c4aaeee57be56309c199c40231 Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Sat, 10 Sep 2022 21:31:41 +0800 Subject: [PATCH 2/7] Fix invalid offset submitted by sinktask #310 --- .../connect/runtime/connectorwrapper/WorkerSinkTask.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTask.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTask.java index 14dc9cbc3..525887702 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTask.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTask.java @@ -374,9 +374,10 @@ private void compareAndCommit(Map offsetsToCommit, Map offsets, int seqno) { log.debug("{} Committing offsets synchronously using sequence number {}: {}", this, seqno, offsets); try { - offsets.forEach((queue, offset) -> { - consumer.getOffsetStore().updateOffset(queue, offset, true); - }); + for (Map.Entry offsetEntry : offsets.entrySet()) { + consumer.getOffsetStore().updateOffset(offsetEntry.getKey(), offsetEntry.getValue(), true); + consumer.getOffsetStore().updateConsumeOffsetToBroker(offsetEntry.getKey(), offsetEntry.getValue(), false); + } onCommitCompleted(null, seqno, offsets); } catch (Exception e) { onCommitCompleted(e, seqno, offsets); From 47f0aacaa989c6fe04ad13667c9093d09a909f88 Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Thu, 2 Mar 2023 11:05:38 +0800 Subject: [PATCH 3/7] Adding the COMPACTION attribute is supported when creating a topic in the BrokerBasedLog #436 --- connectors/rocketmq-connect-activemq/pom.xml | 2 +- connectors/rocketmq-connect-cassandra/pom.xml | 2 +- connectors/rocketmq-connect-debezium/pom.xml | 3 +- .../connect/debezium/RocketMqAdminUtil.java | 90 ++++-- .../debezium/RocketMqDatabaseHistory.java | 2 +- connectors/rocketmq-connect-deltalake/pom.xml | 2 +- connectors/rocketmq-connect-hudi/README.md | 2 +- connectors/rocketmq-connect-hudi/pom.xml | 4 +- connectors/rocketmq-connect-jdbc/pom.xml | 2 +- connectors/rocketmq-connect-jms/pom.xml | 2 +- connectors/rocketmq-connect-kafka/pom.xml | 2 +- connectors/rocketmq-connect-mongo/pom.xml | 2 +- connectors/rocketmq-connect-mqtt/pom.xml | 2 +- connectors/rocketmq-connect-rabbitmq/pom.xml | 2 +- connectors/rocketmq-replicator/pom.xml | 2 +- .../metrics/reporter/RocketMQClientUtil.java | 25 +- .../reporter/RocketMQScheduledReporter.java | 18 +- pom.xml | 3 +- rocketmq-connect-runtime/pom.xml | 3 - .../AbstractStateManagementService.java | 30 +- .../service/ClusterManagementServiceImpl.java | 6 +- .../runtime/stats/ConnectStatsManager.java | 10 +- .../connect/runtime/utils/ConnectUtil.java | 287 ++++++++++++------ .../utils/datasync/BrokerBasedLog.java | 8 +- .../ServerResponseMocker.java | 18 +- .../runtime/connectorwrapper/WorkerTest.java | 37 ++- 26 files changed, 356 insertions(+), 210 deletions(-) diff --git a/connectors/rocketmq-connect-activemq/pom.xml b/connectors/rocketmq-connect-activemq/pom.xml index 738bf7cff..2b5d350f4 100644 --- a/connectors/rocketmq-connect-activemq/pom.xml +++ b/connectors/rocketmq-connect-activemq/pom.xml @@ -207,7 +207,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.2 + 4.9.4 commons-cli diff --git a/connectors/rocketmq-connect-cassandra/pom.xml b/connectors/rocketmq-connect-cassandra/pom.xml index ba30cefe7..4071ab7f6 100644 --- a/connectors/rocketmq-connect-cassandra/pom.xml +++ b/connectors/rocketmq-connect-cassandra/pom.xml @@ -245,7 +245,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.2 + 4.9.4 commons-cli diff --git a/connectors/rocketmq-connect-debezium/pom.xml b/connectors/rocketmq-connect-debezium/pom.xml index 4c7d6c5e8..eabe0d834 100644 --- a/connectors/rocketmq-connect-debezium/pom.xml +++ b/connectors/rocketmq-connect-debezium/pom.xml @@ -180,8 +180,7 @@ 1.7.2.Final 42.3.3 - 4.7.1 - 4.3.2 + 5.1.0 0.1.4 diff --git a/connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/RocketMqAdminUtil.java b/connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/RocketMqAdminUtil.java index c1ae1cbe0..25c595349 100644 --- a/connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/RocketMqAdminUtil.java +++ b/connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/RocketMqAdminUtil.java @@ -16,6 +16,11 @@ */ package org.apache.rocketmq.connect.debezium; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.acl.common.AclClientRPCHook; import org.apache.rocketmq.acl.common.SessionCredentials; @@ -24,28 +29,22 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.admin.TopicOffset; -import org.apache.rocketmq.common.admin.TopicStatsTable; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.body.ClusterInfo; -import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.CommandUtil; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; - /** * Tools for creating RocketMq topic and group */ @@ -130,7 +129,7 @@ public static void createTopic(RocketMqConfig config, TopicConfig topicConfig) { try { defaultMQAdminExt = startMQAdminTool(config); ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - HashMap> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); Set clusterNameSet = clusterAddrTable.keySet(); for (String clusterName : clusterNameSet) { Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); @@ -201,7 +200,7 @@ public static String createSubGroup(RocketMqConfig connectConfig, String subGrou SubscriptionGroupConfig initConfig = new SubscriptionGroupConfig(); initConfig.setGroupName(subGroup); ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - HashMap> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); Set clusterNameSet = clusterAddrTable.keySet(); for (String clusterName : clusterNameSet) { Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); @@ -231,9 +230,9 @@ public static Map offsets(RocketMqConfig config, Stri DefaultMQAdminExt adminClient = null; try { adminClient = RocketMqAdminUtil.startMQAdminTool(config); - TopicStatsTable topicStatsTable = adminClient.examineTopicStats(topic); + TopicStatsTable topicStatsTable = examineTopicStats(adminClient,topic); return topicStatsTable.getOffsetTable(); - } catch (MQClientException | MQBrokerException | RemotingException | InterruptedException e) { + } catch (Exception e) { throw new RuntimeException(e); } finally { if (adminClient != null) { @@ -242,4 +241,59 @@ public static Map offsets(RocketMqConfig config, Stri } } + /** + * Compatible with 4.9.4 and earlier + * + * @param adminClient + * @param topic + * @return + */ + private static TopicStatsTable examineTopicStats(DefaultMQAdminExt adminClient, String topic) { + try { + return adminClient.examineTopicStats(topic); + } catch (MQBrokerException e) { + // Compatible with 4.9.4 and earlier + if (e.getResponseCode() == ResponseCode.REQUEST_CODE_NOT_SUPPORTED) { + try { + return overrideExamineTopicStats(adminClient, topic); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } else { + throw new RuntimeException(e); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + /** + * Compatible with version 4.9.4 + * + * @param adminClient + * @param topic + * @return + * @throws RemotingException + * @throws InterruptedException + * @throws MQClientException + * @throws MQBrokerException + */ + private static TopicStatsTable overrideExamineTopicStats(DefaultMQAdminExt adminClient, + String topic) throws RemotingException, InterruptedException, MQClientException, MQBrokerException { + TopicRouteData topicRouteData = adminClient.examineTopicRouteInfo(topic); + TopicStatsTable topicStatsTable = new TopicStatsTable(); + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + TopicStatsTable tst = adminClient + .getDefaultMQAdminExtImpl() + .getMqClientInstance() + .getMQClientAPIImpl() + .getTopicStatsInfo(addr, topic, 5000); + topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable()); + } + } + return topicStatsTable; + } + } diff --git a/connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/RocketMqDatabaseHistory.java b/connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/RocketMqDatabaseHistory.java index 7fd7941e6..54f8458b0 100644 --- a/connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/RocketMqDatabaseHistory.java +++ b/connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/RocketMqDatabaseHistory.java @@ -34,12 +34,12 @@ import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.connect.kafka.connect.adaptor.task.AbstractKafkaConnectSource; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/connectors/rocketmq-connect-deltalake/pom.xml b/connectors/rocketmq-connect-deltalake/pom.xml index 245395ab2..1333c343b 100644 --- a/connectors/rocketmq-connect-deltalake/pom.xml +++ b/connectors/rocketmq-connect-deltalake/pom.xml @@ -231,7 +231,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.2 + 4.9.4 diff --git a/connectors/rocketmq-connect-hudi/README.md b/connectors/rocketmq-connect-hudi/README.md index ca73a68f9..5a96edd0b 100644 --- a/connectors/rocketmq-connect-hudi/README.md +++ b/connectors/rocketmq-connect-hudi/README.md @@ -72,7 +72,7 @@ http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-hudi-connector-name}/ * **spark-submit 启动任务** 将connect-runtime打包后通过spark-submit提交任务 ``` -nohup sh spark-submit --class org.apache.rocketmq.connect.runtime.DistributedConnectStartup --conf "spark.driver.extraJavaOptions=-Dlogback.configurationFile=logback.xml" --files /xxx/conf/connect.conf,/xxx/conf/log4j.properties --packages org.apache.hudi:hudi-spark3-bundle_2.12:0.8.0,org.apache.spark:spark-avro_2.12:3.0.1,org.apache.hudi:hudi-java-client:0.8.0,org.apache.parquet:parquet-avro:1.10.1,org.apache.avro:avro:1.10.2,com.alibaba:fastjson:1.2.51,org.reflections:reflections:0.9.11,org.apache.httpcomponents:httpclient:4.5.5,io.openmessaging:openmessaging-connector:0.1.1,commons-cli:commons-cli:1.1,org.apache.rocketmq:rocketmq-client:4.4.0,org.apache.rocketmq:rocketmq-tools:4.4.0,org.apache.rocketmq:rocketmq-remoting:4.4.0,org.apache.rocketmq:rocketmq-openmessaging:4.3.2,org.slf4j:slf4j-api:1.7.7,com.google.guava:guava:20.0,org.apache.hadoop:hadoop-common:3.3.1,org.reflections:reflections:0.9.12,org.apache.hive:hive-exec:2.3.7 --conf 'spark.executor.userClassPathFirst=true' --conf 'spark.driver.userClassPathFirst=true' --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' /xxx/rocketmq/rocketmq-connect-runtime-0.0.1-SNAPSHOT.jar & +nohup sh spark-submit --class org.apache.rocketmq.connect.runtime.DistributedConnectStartup --conf "spark.driver.extraJavaOptions=-Dlogback.configurationFile=logback.xml" --files /xxx/conf/connect.conf,/xxx/conf/log4j.properties --packages org.apache.hudi:hudi-spark3-bundle_2.12:0.8.0,org.apache.spark:spark-avro_2.12:3.0.1,org.apache.hudi:hudi-java-client:0.8.0,org.apache.parquet:parquet-avro:1.10.1,org.apache.avro:avro:1.10.2,com.alibaba:fastjson:1.2.51,org.reflections:reflections:0.9.11,org.apache.httpcomponents:httpclient:4.5.5,io.openmessaging:openmessaging-connector:0.1.1,commons-cli:commons-cli:1.1,org.apache.rocketmq:rocketmq-client:4.4.0,org.apache.rocketmq:rocketmq-tools:4.4.0,org.apache.rocketmq:rocketmq-remoting:4.4.0,org.apache.rocketmq:rocketmq-openmessaging:4.9.4,org.slf4j:slf4j-api:1.7.7,com.google.guava:guava:20.0,org.apache.hadoop:hadoop-common:3.3.1,org.reflections:reflections:0.9.12,org.apache.hive:hive-exec:2.3.7 --conf 'spark.executor.userClassPathFirst=true' --conf 'spark.driver.userClassPathFirst=true' --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' /xxx/rocketmq/rocketmq-connect-runtime-0.0.1-SNAPSHOT.jar & ``` 后续操作参考rocketmq-connect-hudi启动步骤 diff --git a/connectors/rocketmq-connect-hudi/pom.xml b/connectors/rocketmq-connect-hudi/pom.xml index a8d03d122..b5bddecd7 100644 --- a/connectors/rocketmq-connect-hudi/pom.xml +++ b/connectors/rocketmq-connect-hudi/pom.xml @@ -15,7 +15,7 @@ 1.8 1.8 - 4.7.1 + 4.9.4 0.8.0 1.10.2 @@ -231,7 +231,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.2 + 4.9.4 org.apache.hudi diff --git a/connectors/rocketmq-connect-jdbc/pom.xml b/connectors/rocketmq-connect-jdbc/pom.xml index 1d7db045c..d7635e33a 100644 --- a/connectors/rocketmq-connect-jdbc/pom.xml +++ b/connectors/rocketmq-connect-jdbc/pom.xml @@ -46,7 +46,7 @@ 1.8 1.8 - 4.7.1 + 4.9.4 4.13.1 diff --git a/connectors/rocketmq-connect-jms/pom.xml b/connectors/rocketmq-connect-jms/pom.xml index 329bbf004..3b3967a42 100644 --- a/connectors/rocketmq-connect-jms/pom.xml +++ b/connectors/rocketmq-connect-jms/pom.xml @@ -188,7 +188,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.2 + 4.9.4 commons-cli diff --git a/connectors/rocketmq-connect-kafka/pom.xml b/connectors/rocketmq-connect-kafka/pom.xml index 2d2e0c2dc..83af208c3 100644 --- a/connectors/rocketmq-connect-kafka/pom.xml +++ b/connectors/rocketmq-connect-kafka/pom.xml @@ -179,7 +179,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.2 + 4.9.4 com.alibaba diff --git a/connectors/rocketmq-connect-mongo/pom.xml b/connectors/rocketmq-connect-mongo/pom.xml index 9f9c1be5a..c94ace678 100644 --- a/connectors/rocketmq-connect-mongo/pom.xml +++ b/connectors/rocketmq-connect-mongo/pom.xml @@ -196,7 +196,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.2 + 4.9.4 junit diff --git a/connectors/rocketmq-connect-mqtt/pom.xml b/connectors/rocketmq-connect-mqtt/pom.xml index 69003ebe0..8ccd0ffb3 100644 --- a/connectors/rocketmq-connect-mqtt/pom.xml +++ b/connectors/rocketmq-connect-mqtt/pom.xml @@ -32,7 +32,7 @@ 1.8 1.8 - 4.7.1 + 4.9.4 diff --git a/connectors/rocketmq-connect-rabbitmq/pom.xml b/connectors/rocketmq-connect-rabbitmq/pom.xml index 64b7a40d1..d53592d05 100644 --- a/connectors/rocketmq-connect-rabbitmq/pom.xml +++ b/connectors/rocketmq-connect-rabbitmq/pom.xml @@ -206,7 +206,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.2 + 4.9.4 commons-cli diff --git a/connectors/rocketmq-replicator/pom.xml b/connectors/rocketmq-replicator/pom.xml index 15b716539..1ef16e7d9 100644 --- a/connectors/rocketmq-replicator/pom.xml +++ b/connectors/rocketmq-replicator/pom.xml @@ -67,7 +67,7 @@ - 4.7.1 + 4.9.4 0.1.3 4.13.1 3.2.4 diff --git a/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQClientUtil.java b/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQClientUtil.java index e3587d693..114bd3965 100644 --- a/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQClientUtil.java +++ b/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQClientUtil.java @@ -17,26 +17,25 @@ package org.apache.rocketmq.connect.metrics.reporter; -import com.beust.jcommander.internal.Sets; +import com.google.common.collect.Sets; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import org.apache.rocketmq.acl.common.AclClientRPCHook; import org.apache.rocketmq.acl.common.SessionCredentials; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.protocol.body.ClusterInfo; -import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.CommandUtil; -import java.util.HashMap; -import java.util.Set; -import java.util.UUID; - /** * rocket connect util @@ -59,7 +58,7 @@ public static DefaultMQProducer initDefaultMQProducer(boolean aclEnabled, String namesrvAddr) { RPCHook rpcHook = null; if (aclEnabled) { - rpcHook = new AclClientRPCHook(new SessionCredentials(accessKey, secretKey)); + rpcHook = getAclRPCHook(accessKey, secretKey); } DefaultMQProducer producer = new DefaultMQProducer(rpcHook); producer.setNamesrvAddr(namesrvAddr); @@ -94,7 +93,7 @@ public static void createTopic(DefaultMQAdminExt defaultMQAdminExt, TopicConfig topicConfig) { try { ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - HashMap> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); Set clusterNameSet = clusterAddrTable.keySet(); for (String clusterName : clusterNameSet) { Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); @@ -140,7 +139,7 @@ public static String createSubGroup(DefaultMQAdminExt defaultMQAdminExt, String SubscriptionGroupConfig initConfig = new SubscriptionGroupConfig(); initConfig.setGroupName(subGroup); ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - HashMap> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); Set clusterNameSet = clusterAddrTable.keySet(); for (String clusterName : clusterNameSet) { Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); diff --git a/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQScheduledReporter.java b/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQScheduledReporter.java index bc504c44e..29229d993 100644 --- a/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQScheduledReporter.java +++ b/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQScheduledReporter.java @@ -16,9 +16,12 @@ */ package org.apache.rocketmq.connect.metrics.reporter; - import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.SortedMap; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.message.Message; @@ -28,11 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.SortedMap; -import java.util.concurrent.TimeUnit; - /** * rocketmq exporter */ @@ -160,7 +158,9 @@ private void send(MetricName name, Double value) { message.setTopic(this.topic); message.setKeys(name.getStr()); message.setBody(value.toString().getBytes(StandardCharsets.UTF_8)); - producer.send(message); + if (producer != null) { + producer.send(message); + } } catch (Exception e) { log.error("Send metrics error", e); } @@ -169,6 +169,8 @@ private void send(MetricName name, Double value) { @Override public void close() { super.close(); - producer.shutdown(); + if (producer != null) { + producer.shutdown(); + } } } diff --git a/pom.xml b/pom.xml index 2429e8b1d..3a91a0a08 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,6 @@ - 4.7.1 4.13.1 3.22.0 4.7.0 @@ -57,7 +56,7 @@ 2.13.4.1 4.4 - 4.7.1 + 5.1.0 3.8.1 UTF-8 diff --git a/rocketmq-connect-runtime/pom.xml b/rocketmq-connect-runtime/pom.xml index 82bd7a610..4ed44af40 100644 --- a/rocketmq-connect-runtime/pom.xml +++ b/rocketmq-connect-runtime/pom.xml @@ -42,9 +42,6 @@ 1.8 1.8 - - 4.7.1 - diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractStateManagementService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractStateManagementService.java index 153e53791..edad9f188 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractStateManagementService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractStateManagementService.java @@ -21,33 +21,29 @@ import io.openmessaging.connector.api.data.SchemaAndValue; import io.openmessaging.connector.api.data.SchemaBuilder; import io.openmessaging.connector.api.data.Struct; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.rocketmq.connect.runtime.common.ConnAndTaskStatus; import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.status.AbstractStatus; import org.apache.rocketmq.connect.runtime.connectorwrapper.status.ConnectorStatus; import org.apache.rocketmq.connect.runtime.connectorwrapper.status.TaskStatus; -import org.apache.rocketmq.connect.runtime.serialization.Serdes; import org.apache.rocketmq.connect.runtime.store.KeyValueStore; import org.apache.rocketmq.connect.runtime.utils.Callback; -import org.apache.rocketmq.connect.runtime.utils.ConnectUtil; import org.apache.rocketmq.connect.runtime.utils.ConnectorTaskId; import org.apache.rocketmq.connect.runtime.utils.Utils; -import org.apache.rocketmq.connect.runtime.utils.datasync.BrokerBasedLog; import org.apache.rocketmq.connect.runtime.utils.datasync.DataSynchronizer; import org.apache.rocketmq.connect.runtime.utils.datasync.DataSynchronizerCallback; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - /** * State management service */ @@ -100,15 +96,7 @@ public void initialize(WorkerConfig config, RecordConverter converter) { this.converter.configure(new HashMap<>()); this.statusTopic = config.getConnectStatusTopic(); this.dataSynchronizer = initializationDataSynchronizer(config); - - new BrokerBasedLog(config, - this.statusTopic, - ConnectUtil.createGroupName(statusManagePrefix, config.getWorkerId()), - new StatusChangeCallback(), - Serdes.serdeFrom(String.class), - Serdes.serdeFrom(byte[].class), - enabledCompactTopic() - ); + } @Override diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImpl.java index 95be089ad..6aeca0eec 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImpl.java @@ -22,14 +22,14 @@ import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.protocol.RequestCode; -import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.utils.ConnectUtil; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -161,7 +161,7 @@ public RemotingCommand workerChanged(ChannelHandlerContext ctx, workerChangeListener.onWorkerChange(); } } catch (Exception e) { - log.error("NotifyConsumerIdsChanged for connect exception", RemotingHelper.exceptionSimpleDesc(e)); + log.error("NotifyConsumerIdsChanged for connect exception", e); } return null; } diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsManager.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsManager.java index 854e6a832..e4450749e 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsManager.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsManager.java @@ -18,17 +18,16 @@ package org.apache.rocketmq.connect.runtime.stats; import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.stats.StatsItem; import org.apache.rocketmq.common.stats.StatsItemSet; import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import java.util.HashMap; import java.util.List; import java.util.concurrent.ScheduledExecutorService; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import static org.apache.rocketmq.connect.runtime.common.LoggerName.ROCKETMQ_CONNECT_STATS; @@ -76,11 +75,8 @@ public class ConnectStatsManager { /** * read disk follow stats */ - private static final InternalLogger log = InternalLoggerFactory.getLogger(ROCKETMQ_CONNECT_STATS); + private static final Logger log = LoggerFactory.getLogger(ROCKETMQ_CONNECT_STATS); - - private static final InternalLogger COMMERCIAL_LOG = InternalLoggerFactory.getLogger( - LoggerName.COMMERCIAL_LOGGER_NAME); private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor("ConnectStatsThread", true); private final ScheduledExecutorService commercialExecutor = diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java index d65849a36..e5321fe3f 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java @@ -17,10 +17,19 @@ package org.apache.rocketmq.connect.runtime.utils; -import com.beust.jcommander.internal.Sets; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import io.openmessaging.connector.api.data.RecordOffset; import io.openmessaging.connector.api.data.RecordPartition; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.acl.common.AclClientRPCHook; import org.apache.rocketmq.acl.common.SessionCredentials; @@ -30,54 +39,45 @@ import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.admin.ConsumeStats; -import org.apache.rocketmq.common.admin.OffsetWrapper; -import org.apache.rocketmq.common.admin.TopicOffset; -import org.apache.rocketmq.common.admin.TopicStatsTable; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.common.protocol.body.ClusterInfo; -import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; -import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; +import org.apache.rocketmq.common.utils.NetworkUtil; +import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.service.strategy.AllocateConnAndTaskStrategy; import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.CommandUtil; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.apache.rocketmq.connect.runtime.connectorwrapper.WorkerSinkTask.QUEUE_OFFSET; public class ConnectUtil { - + private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_RUNTIME); public static final String SYS_TASK_CG_PREFIX = "connect-"; - private final static AtomicLong GROUP_POSTFIX_ID = new AtomicLong(0); public static String createGroupName(String prefix) { StringBuilder sb = new StringBuilder(); sb.append(prefix).append("-"); - sb.append(RemotingUtil.getLocalAddress()).append("-"); + sb.append(NetworkUtil.getLocalAddress()).append("-"); sb.append(UtilAll.getPid()).append("-"); sb.append(System.nanoTime()); return sb.toString().replace(".", "-"); @@ -177,7 +177,7 @@ public static void createTopic(WorkerConfig connectConfig, TopicConfig topicConf try { defaultMQAdminExt = startMQAdminTool(connectConfig); ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - HashMap> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); Set clusterNameSet = clusterAddrTable.keySet(); for (String clusterName : clusterNameSet) { Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); @@ -248,7 +248,7 @@ public static String createSubGroup(WorkerConfig connectConfig, String subGroup) SubscriptionGroupConfig initConfig = new SubscriptionGroupConfig(); initConfig.setGroupName(subGroup); ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - HashMap> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); Set clusterNameSet = clusterAddrTable.keySet(); for (String clusterName : clusterNameSet) { Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); @@ -347,26 +347,6 @@ private static RPCHook getAclRPCHook(String accessKey, String secretKey) { return new AclClientRPCHook(new SessionCredentials(accessKey, secretKey)); } - public static DefaultMQPullConsumer initDefaultMQPullConsumer(WorkerConfig connectConfig, ConnectorTaskId id, ConnectKeyValue keyValue) { - RPCHook rpcHook = null; - if (connectConfig.getAclEnable()) { - rpcHook = new AclClientRPCHook(new SessionCredentials(connectConfig.getAccessKey(), connectConfig.getSecretKey())); - } - DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(rpcHook); - consumer.setInstanceName(id.toString()); - String taskGroupId = keyValue.getString("task-group-id"); - if (StringUtils.isNotBlank(taskGroupId)) { - consumer.setConsumerGroup(taskGroupId); - } else { - consumer.setConsumerGroup(SYS_TASK_CG_PREFIX + id.connector()); - } - if (StringUtils.isNotBlank(connectConfig.getNamesrvAddr())) { - consumer.setNamesrvAddr(connectConfig.getNamesrvAddr()); - } - return consumer; - } - - /** * Get topic offsets */ @@ -377,14 +357,11 @@ public static Map> offsetTopics( try { adminClient = startMQAdminTool(config); for (String topic : topics) { - TopicStatsTable topicStatsTable = adminClient.examineTopicStats(topic); + TopicStatsTable topicStatsTable = examineTopicStats(adminClient, topic); offsets.put(topic, topicStatsTable.getOffsetTable()); } return offsets; - } catch (MQClientException - | MQBrokerException - | RemotingException - | InterruptedException e) { + } catch (Exception e) { throw new RuntimeException(e); } finally { if (adminClient != null) { @@ -393,40 +370,6 @@ public static Map> offsetTopics( } } - /** Flat topics offsets */ - public static Map flatOffsetTopics( - WorkerConfig config, List topics) { - Map messageQueueTopicOffsets = Maps.newConcurrentMap(); - offsetTopics(config, topics).values() - .forEach( - offsetTopic -> { - messageQueueTopicOffsets.putAll(offsetTopic); - }); - return messageQueueTopicOffsets; - } - - /** Search offsets by timestamp */ - public static Map searchOffsetsByTimestamp( - WorkerConfig config, - Collection messageQueues, - Long timestamp) { - Map offsets = Maps.newConcurrentMap(); - DefaultMQAdminExt adminClient = null; - try { - adminClient = startMQAdminTool(config); - for (MessageQueue messageQueue : messageQueues) { - long offset = adminClient.searchOffset(messageQueue, timestamp); - offsets.put(messageQueue, offset); - } - return offsets; - } catch (MQClientException e) { - throw new RuntimeException(e); - } finally { - if (adminClient != null) { - adminClient.shutdown(); - } - } - } /** Get consumer group offset */ public static Map currentOffsets(WorkerConfig config, String groupName, List topics, Set messageQueues) { @@ -436,7 +379,7 @@ public static Map currentOffsets(WorkerConfig config, String adminClient = startMQAdminTool(config); Map consumerOffsets = Maps.newConcurrentMap(); for (String topic : topics) { - ConsumeStats consumeStats = adminClient.examineConsumeStats(groupName, topic); + ConsumeStats consumeStats = examineConsumeStats(adminClient, groupName, topic); consumerOffsets.putAll(consumeStats.getOffsetTable()); } return consumerOffsets.keySet().stream() @@ -446,12 +389,9 @@ public static Map currentOffsets(WorkerConfig config, String messageQueue -> messageQueue, messageQueue -> consumerOffsets.get(messageQueue).getConsumerOffset())); - } catch (MQClientException - | MQBrokerException - | RemotingException - | InterruptedException e) { + } catch (MQClientException e) { if (e instanceof MQClientException) { - if (((MQClientException) e).getResponseCode() == ResponseCode.TOPIC_NOT_EXIST) { + if (e.getResponseCode() == ResponseCode.TOPIC_NOT_EXIST) { return Collections.emptyMap(); } else { throw new RuntimeException(e); @@ -466,4 +406,161 @@ public static Map currentOffsets(WorkerConfig config, String } } + /** + * Compatible with 4.9.4 and earlier + * + * @param adminClient + * @param topic + * @return + */ + private static TopicStatsTable examineTopicStats(DefaultMQAdminExt adminClient, String topic) { + try { + return adminClient.examineTopicStats(topic); + } catch (MQBrokerException e) { + // Compatible with 4.9.4 and earlier + if (e.getResponseCode() == ResponseCode.REQUEST_CODE_NOT_SUPPORTED) { + try { + log.warn("Examine topic stats failure , the server version is less than 5.1.0, and downward compatibility begins, {}", e.getErrorMessage()); + return overrideExamineTopicStats(adminClient, topic); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } else { + throw new RuntimeException(e); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + /** + * examineConsumeStats + * Compatible with 4.9.4 and earlier + * + * @param adminClient + * @param topic + * @return + */ + private static ConsumeStats examineConsumeStats(DefaultMQAdminExt adminClient, String groupName, String topic) { + try { + return adminClient.examineConsumeStats(groupName, topic); + } catch (MQBrokerException e) { + // Compatible with 4.9.4 and earlier + if (e.getResponseCode() == ResponseCode.REQUEST_CODE_NOT_SUPPORTED) { + try { + log.warn("Examine consume stats failure, the server version is less than 5.1.0, and downward compatibility begins {}", e.getErrorMessage()); + return overrideExamineConsumeStats(adminClient, groupName, topic); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } else { + throw new RuntimeException(e); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + /** + * Compatible with version 4.9.4 + * + * @param adminClient + * @param topic + * @return + * @throws RemotingException + * @throws InterruptedException + * @throws MQClientException + * @throws MQBrokerException + */ + private static TopicStatsTable overrideExamineTopicStats(DefaultMQAdminExt adminClient, + String topic) throws RemotingException, InterruptedException, MQClientException, MQBrokerException { + TopicRouteData topicRouteData = adminClient.examineTopicRouteInfo(topic); + TopicStatsTable topicStatsTable = new TopicStatsTable(); + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + TopicStatsTable tst = adminClient + .getDefaultMQAdminExtImpl() + .getMqClientInstance() + .getMQClientAPIImpl() + .getTopicStatsInfo(addr, topic, 5000); + topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable()); + } + } + return topicStatsTable; + } + + /** + * Compatible with version 4.9.4 + * + * @param adminExt + * @param groupName + * @param topic + * @return + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + * @throws MQBrokerException + */ + private static ConsumeStats overrideExamineConsumeStats(DefaultMQAdminExt adminExt, String groupName, + String topic) throws MQClientException, RemotingException, InterruptedException, MQBrokerException { + TopicRouteData topicRouteData = null; + List routeTopics = new ArrayList<>(); + routeTopics.add(MixAll.getRetryTopic(groupName)); + if (topic != null) { + routeTopics.add(topic); + routeTopics.add(KeyBuilder.buildPopRetryTopic(topic, groupName)); + } + for (int i = 0; i < routeTopics.size(); i++) { + try { + topicRouteData = adminExt.getDefaultMQAdminExtImpl().examineTopicRouteInfo(routeTopics.get(i)); + if (topicRouteData != null) { + break; + } + } catch (Throwable e) { + if (i == routeTopics.size() - 1) { + throw e; + } + } + } + ConsumeStats result = new ConsumeStats(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + ConsumeStats consumeStats = adminExt.getDefaultMQAdminExtImpl().getMqClientInstance().getMQClientAPIImpl().getConsumeStats(addr, groupName, topic, 5000 * 3); + result.getOffsetTable().putAll(consumeStats.getOffsetTable()); + double value = result.getConsumeTps() + consumeStats.getConsumeTps(); + result.setConsumeTps(value); + } + } + + Set topics = Sets.newHashSet(); + for (MessageQueue messageQueue : result.getOffsetTable().keySet()) { + topics.add(messageQueue.getTopic()); + } + + ConsumeStats staticResult = new ConsumeStats(); + staticResult.setConsumeTps(result.getConsumeTps()); + + for (String currentTopic : topics) { + TopicRouteData currentRoute = adminExt.getDefaultMQAdminExtImpl().examineTopicRouteInfo(currentTopic); + if (currentRoute.getTopicQueueMappingByBroker() == null + || currentRoute.getTopicQueueMappingByBroker().isEmpty()) { + //normal topic + for (Map.Entry entry : result.getOffsetTable().entrySet()) { + if (entry.getKey().getTopic().equals(currentTopic)) { + staticResult.getOffsetTable().put(entry.getKey(), entry.getValue()); + } + } + } + } + + if (staticResult.getOffsetTable().isEmpty()) { + throw new MQClientException(ResponseCode.CONSUMER_NOT_ONLINE, "Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message"); + } + + return staticResult; + } + } diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLog.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLog.java index 258adeeb6..01eda1526 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLog.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLog.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.connect.runtime.utils.datasync; +import com.google.common.collect.Maps; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -34,7 +35,6 @@ import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.selector.SelectMessageQueueByHash; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; @@ -46,6 +46,7 @@ import org.apache.rocketmq.connect.runtime.utils.Callback; import org.apache.rocketmq.connect.runtime.utils.ConnectUtil; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -137,6 +138,11 @@ private void prepare() { if (!ConnectUtil.isTopicExist(workerConfig, topicName)) { log.info("Try to create store topic: {}!", topicName); TopicConfig topicConfig = new TopicConfig(topicName, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE); + if (enabledCompactTopic) { + Map attributes = Maps.newConcurrentMap(); + attributes.put("+cleanup.policy", "COMPACTION"); + topicConfig.setAttributes(attributes); + } ConnectUtil.createTopic(workerConfig, topicConfig); } } diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/ServerResponseMocker.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/ServerResponseMocker.java index 16f3ad403..db2fb2aa6 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/ServerResponseMocker.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/ServerResponseMocker.java @@ -30,10 +30,7 @@ import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.timeout.IdleStateHandler; import io.netty.util.concurrent.Future; -import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.protocol.RequestCode; -import org.apache.rocketmq.common.protocol.body.ClusterInfo; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; @@ -42,17 +39,20 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; -import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody; -import org.apache.rocketmq.common.protocol.route.BrokerData; import java.util.concurrent.ConcurrentHashMap; -import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; -import org.apache.rocketmq.common.protocol.route.QueueData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.remoting.netty.NettyDecoder; import org.apache.rocketmq.remoting.netty.NettyEncoder; +import org.apache.rocketmq.remoting.protocol.DataVersion; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.QueueData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.junit.After; import org.junit.Before; diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerTest.java index 5570be210..01ac4735e 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerTest.java @@ -17,9 +17,20 @@ package org.apache.rocketmq.connect.runtime.connectorwrapper; +import com.google.common.collect.Sets; import io.openmessaging.connector.api.component.connector.ConnectorContext; import io.openmessaging.connector.api.data.ConnectRecord; import io.openmessaging.internal.DefaultKeyValue; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; @@ -45,28 +56,21 @@ import org.apache.rocketmq.connect.runtime.service.local.LocalStateManagementServiceImpl; import org.apache.rocketmq.connect.runtime.stats.ConnectStatsManager; import org.apache.rocketmq.connect.runtime.stats.ConnectStatsService; +import org.apache.rocketmq.connect.runtime.utils.ConnectUtil; import org.apache.rocketmq.connect.runtime.utils.ConnectorTaskId; import org.apache.rocketmq.connect.runtime.utils.TestUtils; +import org.apache.rocketmq.connect.runtime.utils.datasync.DataSynchronizer; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.junit.MockitoJUnitRunner; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -75,9 +79,6 @@ public class WorkerTest { @Mock private PositionManagementService positionManagementService; - @Mock - private PositionManagementService offsetManagementService; - @Mock private ConfigManagementService configManagementService; @@ -111,12 +112,18 @@ public class WorkerTest { private WrapperStatusListener wrapperStatusListener; + @Mock private StateManagementService stateManagementService; private ServerResponseMocker nameServerMocker; private ServerResponseMocker brokerMocker; + @Mock + protected DataSynchronizer dataSynchronizer; + + private final MockedStatic connectUtil = mockStatic(ConnectUtil.class); + @Before public void init() { nameServerMocker = NameServerMocker.startByDefaultConf(9876, 10911); @@ -125,6 +132,7 @@ public void init() { when(plugin.newConnector(any())).thenReturn(new TestConnector()); when(plugin.delegatingLoader()).thenReturn(delegatingClassLoader); when(delegatingClassLoader.pluginClassLoader(any())).thenReturn(pluginClassLoader); + connectUtil.when(() -> ConnectUtil.fetchAllConsumerGroupList(connectConfig)).thenReturn(Sets.newHashSet()); Thread.currentThread().setContextClassLoader(pluginClassLoader); connectConfig = new WorkerConfig(); @@ -179,6 +187,7 @@ public void init() { @After public void destroy() throws InterruptedException { + connectUtil.close(); TimeUnit.SECONDS.sleep(2); worker.stop(); TestUtils.deleteFile(new File(System.getProperty("user.home") + File.separator + "testConnectorStore")); From cc040b272cc249b92db2d2a211d2dd44ffa4d971 Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Tue, 7 Mar 2023 21:09:43 +0800 Subject: [PATCH 4/7] fixed --- .../apache/rocketmq/connect/runtime/stats/ConnectStats.java | 6 +++--- .../rocketmq/connect/runtime/stats/ConnectStatsService.java | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStats.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStats.java index ed462cb4f..205598712 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStats.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStats.java @@ -18,11 +18,11 @@ package org.apache.rocketmq.connect.runtime.stats; import org.apache.rocketmq.connect.runtime.common.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public class ConnectStats { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_CONNECT_STATS); + private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_CONNECT_STATS); private final ConnectStatsService connectStatsService; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java index d7c3c6520..5e52d338d 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java @@ -31,9 +31,11 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public class ConnectStatsService extends ServiceThread { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_CONNECT_STATS); + private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_CONNECT_STATS); private static final int FREQUENCY_OF_SAMPLING = 1000; From 1f5080b31a69e918a30753a16994cb7f666acd91 Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Tue, 7 Mar 2023 21:12:57 +0800 Subject: [PATCH 5/7] upgrade rocketmq client version to 5.1.0 --- connectors/rocketmq-connect-cassandra/pom.xml | 2 +- connectors/rocketmq-connect-deltalake/pom.xml | 2 +- connectors/rocketmq-connect-redis/pom.xml | 2 +- .../rocketmq/connect/runtime/config/WorkerConfig.java | 8 +------- .../connect/runtime/stats/ConnectStatsService.java | 2 -- .../rocketmq/connect/runtime/utils/ConnectUtil.java | 1 - .../runtime/connectorwrapper/NameServerMocker.java | 4 ++-- .../runtime/service/ClusterManagementServiceImplTest.java | 4 ++-- .../runtime/service/ConfigManagementServiceImplTest.java | 4 +--- .../runtime/service/DefaultConnectorContextTest.java | 1 - .../service/PositionManagementServiceImplTest.java | 5 +---- .../runtime/store/PositionStorageReaderImplTest.java | 2 +- .../runtime/utils/datasync/BrokerBasedLogTest.java | 2 +- 13 files changed, 12 insertions(+), 27 deletions(-) diff --git a/connectors/rocketmq-connect-cassandra/pom.xml b/connectors/rocketmq-connect-cassandra/pom.xml index 4071ab7f6..c3c018f52 100644 --- a/connectors/rocketmq-connect-cassandra/pom.xml +++ b/connectors/rocketmq-connect-cassandra/pom.xml @@ -38,7 +38,7 @@ 1.8 1.8 - 4.5.2 + 4.9.4 diff --git a/connectors/rocketmq-connect-deltalake/pom.xml b/connectors/rocketmq-connect-deltalake/pom.xml index 1333c343b..6d96412c5 100644 --- a/connectors/rocketmq-connect-deltalake/pom.xml +++ b/connectors/rocketmq-connect-deltalake/pom.xml @@ -15,7 +15,7 @@ 1.8 1.8 - 4.5.2 + 4.9.4 1.10.2 1.10.1 2.12 diff --git a/connectors/rocketmq-connect-redis/pom.xml b/connectors/rocketmq-connect-redis/pom.xml index 73e3db969..4bd556a68 100644 --- a/connectors/rocketmq-connect-redis/pom.xml +++ b/connectors/rocketmq-connect-redis/pom.xml @@ -16,7 +16,7 @@ 1.8 1.8 - 4.5.2 + 4.9.4 diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/config/WorkerConfig.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/config/WorkerConfig.java index 1a0ee17eb..a29152dfb 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/config/WorkerConfig.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/config/WorkerConfig.java @@ -17,15 +17,10 @@ package org.apache.rocketmq.connect.runtime.config; -import org.apache.rocketmq.common.MixAll; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.util.HashMap; import java.util.Map; - -import static org.apache.rocketmq.connect.runtime.common.LoggerName.ROCKETMQ_RUNTIME; +import org.apache.rocketmq.common.MixAll; /** * Configurations for runtime. @@ -34,7 +29,6 @@ public class WorkerConfig { public static final String METRIC_CLASS = "metrics.reporter"; public static final String CONNECT_HOME_PROPERTY = "connect.home.dir"; public static final String CONNECT_HOME_ENV = "CONNECT_HOME"; - private static final Logger log = LoggerFactory.getLogger(ROCKETMQ_RUNTIME); private String connectHome = System.getProperty(CONNECT_HOME_PROPERTY, System.getenv(CONNECT_HOME_ENV)); /** diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java index 5e52d338d..a30259a8d 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java @@ -20,8 +20,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.connect.runtime.common.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import java.text.MessageFormat; import java.util.HashMap; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java index e5321fe3f..158d6ada4 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java @@ -559,7 +559,6 @@ private static ConsumeStats overrideExamineConsumeStats(DefaultMQAdminExt adminE if (staticResult.getOffsetTable().isEmpty()) { throw new MQClientException(ResponseCode.CONSUMER_NOT_ONLINE, "Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message"); } - return staticResult; } diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/NameServerMocker.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/NameServerMocker.java index a437d5a9c..28339be90 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/NameServerMocker.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/connectorwrapper/NameServerMocker.java @@ -17,12 +17,12 @@ package org.apache.rocketmq.connect.runtime.connectorwrapper; import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; /** * tools class diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImplTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImplTest.java index 269082083..c1cb0051f 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImplTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImplTest.java @@ -21,13 +21,13 @@ import io.netty.channel.ChannelHandlerContext; import java.nio.charset.StandardCharsets; import java.util.List; -import org.apache.rocketmq.common.protocol.RequestCode; -import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.NameServerMocker; import org.apache.rocketmq.connect.runtime.connectorwrapper.ServerResponseMocker; import org.apache.rocketmq.remoting.RemotingClient; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/ConfigManagementServiceImplTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/ConfigManagementServiceImplTest.java index faa60ef8e..9f003999d 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/ConfigManagementServiceImplTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/ConfigManagementServiceImplTest.java @@ -29,12 +29,9 @@ import java.util.Set; import java.util.UUID; -import com.google.common.collect.Maps; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs; @@ -53,6 +50,7 @@ import org.apache.rocketmq.connect.runtime.utils.datasync.BrokerBasedLog; import org.apache.rocketmq.connect.runtime.utils.datasync.DataSynchronizer; import org.apache.rocketmq.connect.runtime.utils.datasync.DataSynchronizerCallback; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/DefaultConnectorContextTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/DefaultConnectorContextTest.java index beed39be8..27b443791 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/DefaultConnectorContextTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/DefaultConnectorContextTest.java @@ -40,7 +40,6 @@ import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/PositionManagementServiceImplTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/PositionManagementServiceImplTest.java index dc1535773..c78a80f98 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/PositionManagementServiceImplTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/service/PositionManagementServiceImplTest.java @@ -19,16 +19,12 @@ import com.google.common.collect.Lists; import io.netty.util.internal.ConcurrentSet; -import io.openmessaging.Future; import io.openmessaging.connector.api.data.RecordOffset; -import io.openmessaging.producer.SendResult; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter; import org.apache.rocketmq.connect.runtime.connectorwrapper.NameServerMocker; @@ -40,6 +36,7 @@ import org.apache.rocketmq.connect.runtime.utils.TestUtils; import org.apache.rocketmq.connect.runtime.utils.datasync.BrokerBasedLog; import org.apache.rocketmq.connect.runtime.utils.datasync.DataSynchronizerCallback; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.assertj.core.util.Maps; import org.junit.After; import org.junit.Before; diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/store/PositionStorageReaderImplTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/store/PositionStorageReaderImplTest.java index 45f64cdd8..a9fdf9e72 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/store/PositionStorageReaderImplTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/store/PositionStorageReaderImplTest.java @@ -24,7 +24,6 @@ import io.openmessaging.connector.api.data.RecordPartition; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.NameServerMocker; @@ -33,6 +32,7 @@ import org.apache.rocketmq.connect.runtime.service.PositionManagementService; import org.apache.rocketmq.connect.runtime.service.local.LocalPositionManagementServiceImpl; import org.apache.rocketmq.connect.runtime.utils.ConnectUtil; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.assertj.core.util.Maps; import org.junit.After; import org.junit.Assert; diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLogTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLogTest.java index b9470c374..a0531db9c 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLogTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLogTest.java @@ -23,7 +23,6 @@ import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.selector.SelectMessageQueueByHash; -import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; @@ -31,6 +30,7 @@ import org.apache.rocketmq.connect.runtime.serialization.Serializer; import org.apache.rocketmq.connect.runtime.utils.ConnectUtil; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.junit.After; import org.junit.Before; import org.junit.Test; From 01d16c8849b9c2fbf8f119f2d54cbaa6afcd73c9 Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Wed, 8 Mar 2023 17:34:26 +0800 Subject: [PATCH 6/7] extend rocketmq-connect-common --- connectors/rocketmq-connect-debezium/pom.xml | 5 + metric-exporter/pom.xml | 5 + .../metrics/reporter/RocketMQClientUtil.java | 157 ------ .../reporter/RocketMQScheduledReporter.java | 38 +- pom.xml | 24 +- rocketmq-connect-common/pom.xml | 34 ++ .../connect/common/ConsumerConfiguration.java | 40 ++ .../connect/common/ProducerConfiguration.java | 39 ++ .../common/RocketMqBaseConfiguration.java | 39 ++ .../connect/common/RocketMqUtils.java | 533 ++++++++++++++++++ .../rocketmq/connect}/common/cache/Cache.java | 2 +- .../connect}/common/cache/LRUCache.java | 2 +- .../connect/common/constant}/LoggerName.java | 2 +- rocketmq-connect-runtime/pom.xml | 5 + .../runtime/DistributedConnectStartup.java | 2 +- .../runtime/StandaloneConnectStartup.java | 2 +- .../connectorwrapper/TransformChain.java | 2 +- .../runtime/connectorwrapper/Worker.java | 2 +- .../connectorwrapper/WorkerDirectTask.java | 2 +- .../connectorwrapper/WorkerSinkTask.java | 2 +- .../WorkerSinkTaskContext.java | 2 +- .../connectorwrapper/WorkerSourceTask.java | 6 +- .../controller/AbstractConnectController.java | 2 +- .../DistributedConnectController.java | 2 +- .../converter/record/json/JsonConverter.java | 6 +- .../errors/DeadLetterQueueReporter.java | 12 +- .../runtime/metrics/ConnectMetrics.java | 2 +- .../rest/ConnectorPluginsResource.java | 2 +- .../connect/runtime/rest/RestHandler.java | 2 +- .../serialization/ListDeserializer.java | 2 +- .../runtime/serialization/ListSerializer.java | 2 +- .../store/ConnectKeyValueDeserializer.java | 2 +- .../store/ConnectKeyValueSerializer.java | 2 +- .../store/RecordOffsetDeserializer.java | 2 +- .../store/RecordOffsetSerializer.java | 2 +- .../store/RecordPartitionDeserializer.java | 2 +- .../store/RecordPartitionSerializer.java | 2 +- .../store/RecordPositionMapDeserializer.java | 2 +- .../store/RecordPositionMapSerializer.java | 2 +- .../AbstractConfigManagementService.java | 115 ++-- .../AbstractPositionManagementService.java | 2 +- .../AbstractStateManagementService.java | 2 +- .../service/ClusterManagementServiceImpl.java | 10 +- .../service/DefaultConnectorContext.java | 2 +- .../runtime/service/RebalanceImpl.java | 4 +- .../runtime/service/RebalanceService.java | 2 +- .../LocalConfigManagementServiceImpl.java | 2 +- .../LocalPositionManagementServiceImpl.java | 2 +- .../LocalStateManagementServiceImpl.java | 2 +- .../FilePositionManagementServiceImpl.java | 2 +- .../MemoryConfigManagementServiceImpl.java | 2 +- .../MemoryStateManagementServiceImpl.java | 2 +- .../memory/StandaloneRebalanceService.java | 2 +- .../RocketMqConfigManagementServiceImpl.java | 2 +- ...RocketMqPositionManagementServiceImpl.java | 2 +- .../RocketMqStateManagementServiceImpl.java | 2 +- ...teConnAndTaskStrategyByConsistentHash.java | 2 +- .../DefaultAllocateConnAndTaskStrategy.java | 2 +- .../connect/runtime/stats/ConnectStats.java | 2 +- .../runtime/stats/ConnectStatsManager.java | 2 +- .../runtime/stats/ConnectStatsService.java | 2 +- .../runtime/store/FileBaseKeyValueStore.java | 2 +- .../runtime/store/PositionStorageWriter.java | 2 +- .../connect/runtime/utils/ConnectUtil.java | 490 +++------------- .../connect/runtime/utils/ServiceThread.java | 2 +- .../utils/datasync/BrokerBasedLog.java | 17 +- .../runtime/common/cache/LRUCacheTest.java | 1 + .../runtime/utils/ConnectUtilTest.java | 49 +- 68 files changed, 949 insertions(+), 774 deletions(-) delete mode 100644 metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQClientUtil.java create mode 100644 rocketmq-connect-common/pom.xml create mode 100644 rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/ConsumerConfiguration.java create mode 100644 rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/ProducerConfiguration.java create mode 100644 rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/RocketMqBaseConfiguration.java create mode 100644 rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/RocketMqUtils.java rename {rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime => rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect}/common/cache/Cache.java (95%) rename {rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime => rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect}/common/cache/LRUCache.java (97%) rename {rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common => rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/constant}/LoggerName.java (95%) diff --git a/connectors/rocketmq-connect-debezium/pom.xml b/connectors/rocketmq-connect-debezium/pom.xml index eabe0d834..affc56325 100644 --- a/connectors/rocketmq-connect-debezium/pom.xml +++ b/connectors/rocketmq-connect-debezium/pom.xml @@ -326,6 +326,11 @@ logback-core 1.2.9 + + org.apache.rocketmq + rocketmq-connect-common + 0.0.1-SNAPSHOT + diff --git a/metric-exporter/pom.xml b/metric-exporter/pom.xml index ca5a56462..ab06e014e 100644 --- a/metric-exporter/pom.xml +++ b/metric-exporter/pom.xml @@ -29,6 +29,7 @@ commons-lang3 3.12.0 + org.apache.rocketmq rocketmq-client @@ -37,6 +38,10 @@ org.apache.rocketmq rocketmq-tools + + org.apache.rocketmq + rocketmq-connect-common + diff --git a/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQClientUtil.java b/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQClientUtil.java deleted file mode 100644 index 114bd3965..000000000 --- a/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQClientUtil.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.connect.metrics.reporter; - -import com.google.common.collect.Sets; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import org.apache.rocketmq.acl.common.AclClientRPCHook; -import org.apache.rocketmq.acl.common.SessionCredentials; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.protocol.LanguageCode; -import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; -import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; -import org.apache.rocketmq.remoting.protocol.route.BrokerData; -import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; -import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.CommandUtil; - - -/** - * rocket connect util - */ -public class RocketMQClientUtil { - - public static String createUniqInstance(String prefix) { - return new StringBuffer(prefix).append("-").append(UUID.randomUUID()).toString(); - } - - private static RPCHook getAclRPCHook(String accessKey, String secretKey) { - return new AclClientRPCHook(new SessionCredentials(accessKey, secretKey)); - } - - - public static DefaultMQProducer initDefaultMQProducer(boolean aclEnabled, - String accessKey, - String secretKey, - String groupId, - String namesrvAddr) { - RPCHook rpcHook = null; - if (aclEnabled) { - rpcHook = getAclRPCHook(accessKey, secretKey); - } - DefaultMQProducer producer = new DefaultMQProducer(rpcHook); - producer.setNamesrvAddr(namesrvAddr); - producer.setInstanceName(createUniqInstance(namesrvAddr)); - producer.setProducerGroup(groupId); - producer.setSendMsgTimeout(5000); - producer.setLanguage(LanguageCode.JAVA); - return producer; - } - - public static DefaultMQAdminExt startMQAdminTool(boolean aclEnabled, - String accessKey, - String secretKey, - String groupId, - String namesrvAddr - ) throws MQClientException { - DefaultMQAdminExt admin; - if (aclEnabled) { - admin = new DefaultMQAdminExt(new AclClientRPCHook(new SessionCredentials(accessKey, secretKey))); - } else { - admin = new DefaultMQAdminExt(); - } - admin.setNamesrvAddr(namesrvAddr); - admin.setAdminExtGroup(groupId); - admin.setInstanceName(createUniqInstance(namesrvAddr)); - admin.start(); - return admin; - } - - - public static void createTopic(DefaultMQAdminExt defaultMQAdminExt, - TopicConfig topicConfig) { - try { - ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); - Set clusterNameSet = clusterAddrTable.keySet(); - for (String clusterName : clusterNameSet) { - Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); - for (String addr : masterSet) { - defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); - } - } - } catch (Exception e) { - throw new RuntimeException("create topic: " + topicConfig.getTopicName() + " failed", e); - } - } - - public static boolean topicExist(DefaultMQAdminExt defaultMQAdminExt, String topic) { - boolean foundTopicRouteInfo = false; - try { - TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); - if (topicRouteData != null) { - foundTopicRouteInfo = true; - } - } catch (Exception e) { - foundTopicRouteInfo = false; - } - return foundTopicRouteInfo; - } - - public static Set fetchAllConsumerGroup(DefaultMQAdminExt defaultMQAdminExt) { - Set consumerGroupSet = Sets.newHashSet(); - try { - ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - for (BrokerData brokerData : clusterInfo.getBrokerAddrTable().values()) { - SubscriptionGroupWrapper subscriptionGroupWrapper = defaultMQAdminExt.getAllSubscriptionGroup(brokerData.selectBrokerAddr(), 3000L); - consumerGroupSet.addAll(subscriptionGroupWrapper.getSubscriptionGroupTable().keySet()); - } - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException("fetch all topic failed", e); - } - return consumerGroupSet; - } - - public static String createSubGroup(DefaultMQAdminExt defaultMQAdminExt, String subGroup) { - try { - SubscriptionGroupConfig initConfig = new SubscriptionGroupConfig(); - initConfig.setGroupName(subGroup); - ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); - Set clusterNameSet = clusterAddrTable.keySet(); - for (String clusterName : clusterNameSet) { - Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); - for (String addr : masterSet) { - defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, initConfig); - } - } - } catch (Exception e) { - throw new RuntimeException("create subGroup: " + subGroup + " failed", e); - } - return subGroup; - } -} - - diff --git a/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQScheduledReporter.java b/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQScheduledReporter.java index 29229d993..f6fe3a7d2 100644 --- a/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQScheduledReporter.java +++ b/metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/reporter/RocketMQScheduledReporter.java @@ -25,9 +25,11 @@ import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.connect.common.ProducerConfiguration; +import org.apache.rocketmq.connect.common.RocketMqBaseConfiguration; +import org.apache.rocketmq.connect.common.RocketMqUtils; import org.apache.rocketmq.connect.metrics.MetricName; import org.apache.rocketmq.connect.metrics.ScheduledMetricsReporter; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,27 +128,35 @@ public void config(Map configs) { } this.topic = configs.get(METRICS_TOPIC); String groupId = configs.get(GROUP_ID); - DefaultMQAdminExt defaultMQAdminExt = null; try { - defaultMQAdminExt = RocketMQClientUtil.startMQAdminTool(Boolean.valueOf(configs.get(ACL_ENABLED)), configs.get(ACCESS_KEY), configs.get(SECRET_KEY), groupId, configs.get(NAMESRV_ADDR)); - if (!RocketMQClientUtil.topicExist(defaultMQAdminExt, topic)) { - RocketMQClientUtil.createTopic(defaultMQAdminExt, new TopicConfig(topic)); + RocketMqBaseConfiguration baseConfiguration = RocketMqBaseConfiguration + .builder() + .namesrvAddr(configs.get(NAMESRV_ADDR)) + .aclEnable(Boolean.valueOf(configs.get(ACL_ENABLED))) + .accessKey(configs.get(ACCESS_KEY)) + .secretKey(configs.get(SECRET_KEY)) + .groupId(groupId) + .build(); + + RocketMqUtils.maybeCreateTopic(baseConfiguration, new TopicConfig(topic)); + if (!RocketMqUtils.fetchAllConsumerGroup(baseConfiguration).contains(groupId)) { + RocketMqUtils.createGroup(baseConfiguration, groupId); } - if (!RocketMQClientUtil.fetchAllConsumerGroup(defaultMQAdminExt).contains(groupId)) { - RocketMQClientUtil.createSubGroup(defaultMQAdminExt, groupId); - } - this.producer = RocketMQClientUtil.initDefaultMQProducer(Boolean.valueOf(configs.get(ACL_ENABLED)), configs.get(ACCESS_KEY), configs.get(SECRET_KEY), groupId, configs.get(NAMESRV_ADDR)); + ProducerConfiguration producerConfiguration = ProducerConfiguration + .producerBuilder() + .namesrvAddr(configs.get(NAMESRV_ADDR)) + .aclEnable(Boolean.valueOf(configs.get(ACL_ENABLED))) + .accessKey(configs.get(ACCESS_KEY)) + .secretKey(configs.get(SECRET_KEY)) + .groupId(groupId) + .build(); + this.producer = RocketMqUtils.initDefaultMQProducer(producerConfiguration); this.producer.start(); } catch (Exception e) { log.error("Init config failed ", e); - } finally { - if (defaultMQAdminExt != null) { - defaultMQAdminExt.shutdown(); - } } } - @Override public void start() { this.start(10, TimeUnit.SECONDS); diff --git a/pom.xml b/pom.xml index 3a91a0a08..cc37e5106 100644 --- a/pom.xml +++ b/pom.xml @@ -23,6 +23,7 @@ pom 0.0.1-SNAPSHOT + rocketmq-connect-common rocketmq-connect-sample rocketmq-connect-runtime rocketmq-connect-cli @@ -57,6 +58,9 @@ 4.4 5.1.0 + + 1.18.0 + 3.8.1 UTF-8 @@ -150,6 +154,11 @@ guava ${guava.version} + + org.apache.rocketmq + rocketmq-connect-common + ${project.version} + org.apache.rocketmq rocketmq-connect-cli @@ -181,21 +190,16 @@ commons-collections4 ${commons-collections4.version} - - org.apache.rocketmq - rocketmq-tools - ${rocketmq.version} - - - org.apache.rocketmq - rocketmq-client - ${rocketmq.version} - org.apache.maven maven-artifact ${maven-artifact.version} + + org.projectlombok + lombok + ${lombok.version} + diff --git a/rocketmq-connect-common/pom.xml b/rocketmq-connect-common/pom.xml new file mode 100644 index 000000000..18511f372 --- /dev/null +++ b/rocketmq-connect-common/pom.xml @@ -0,0 +1,34 @@ + + + + rocketmq-connect + org.apache.rocketmq + 0.0.1-SNAPSHOT + + 4.0.0 + + rocketmq-connect-common + + + 8 + 8 + UTF-8 + + + + org.apache.rocketmq + rocketmq-client + + + org.apache.rocketmq + rocketmq-tools + + + org.projectlombok + lombok + + + + \ No newline at end of file diff --git a/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/ConsumerConfiguration.java b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/ConsumerConfiguration.java new file mode 100644 index 000000000..d559c5152 --- /dev/null +++ b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/ConsumerConfiguration.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.common; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class ConsumerConfiguration extends RocketMqBaseConfiguration { + // consumer + private Integer batchSize; + private Long pollTimeoutMillis; + + + @Builder(builderMethodName = "consumerBuilder") + public ConsumerConfiguration(String namesrvAddr, String groupId, boolean aclEnable, String accessKey, + String secretKey, + Integer batchSize, Long pollTimeoutMillis) { + super(namesrvAddr, groupId, aclEnable, accessKey, secretKey); + this.batchSize = batchSize; + this.pollTimeoutMillis = pollTimeoutMillis; + } +} diff --git a/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/ProducerConfiguration.java b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/ProducerConfiguration.java new file mode 100644 index 000000000..23d8b88a6 --- /dev/null +++ b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/ProducerConfiguration.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.common; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class ProducerConfiguration extends RocketMqBaseConfiguration { + // producer + private Integer maxMessageSize; + private Integer sendMsgTimeout; + + @Builder(builderMethodName = "producerBuilder") + public ProducerConfiguration(String namesrvAddr, String groupId, boolean aclEnable, String accessKey, + String secretKey, + Integer maxMessageSize, Integer sendMsgTimeout) { + super(namesrvAddr, groupId, aclEnable, accessKey, secretKey); + this.maxMessageSize = maxMessageSize; + this.sendMsgTimeout = sendMsgTimeout; + } +} diff --git a/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/RocketMqBaseConfiguration.java b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/RocketMqBaseConfiguration.java new file mode 100644 index 000000000..ac29e36ca --- /dev/null +++ b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/RocketMqBaseConfiguration.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.common; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +/** + * rocketmq base configuration + */ +@Getter +@Setter +@Builder +public class RocketMqBaseConfiguration { + private String namesrvAddr; + private String groupId; + /** + * set acl config + */ + private boolean aclEnable; + private String accessKey; + private String secretKey; +} diff --git a/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/RocketMqUtils.java b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/RocketMqUtils.java new file mode 100644 index 000000000..63c60fc53 --- /dev/null +++ b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/RocketMqUtils.java @@ -0,0 +1,533 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.common; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.connect.common.constant.LoggerName; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.CommandUtil; + +/** + * RocketMq utils + */ +public class RocketMqUtils { + private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_RUNTIME); + + public static String createUniqInstance(String prefix) { + return prefix.concat("-").concat(UUID.randomUUID().toString()); + } + + /** + * Init default mq producer + * + * @param configuration + * @return + */ + public static DefaultMQProducer initDefaultMQProducer(ProducerConfiguration configuration) { + RPCHook rpcHook = null; + if (configuration.isAclEnable()) { + rpcHook = new AclClientRPCHook(new SessionCredentials(configuration.getAccessKey(), configuration.getSecretKey())); + } + DefaultMQProducer producer = new DefaultMQProducer(rpcHook); + producer.setNamesrvAddr(configuration.getNamesrvAddr()); + producer.setInstanceName(createUniqInstance(configuration.getNamesrvAddr())); + producer.setProducerGroup(configuration.getGroupId()); + if (configuration.getSendMsgTimeout() != null) { + producer.setSendMsgTimeout(configuration.getSendMsgTimeout()); + } + if (configuration.getMaxMessageSize() != null) { + producer.setMaxMessageSize(configuration.getMaxMessageSize()); + } + producer.setLanguage(LanguageCode.JAVA); + return producer; + } + + /** + * init default lite pull consumer + * + * @param configuration + * @return + * @throws MQClientException + */ + public static DefaultLitePullConsumer initDefaultLitePullConsumer(ConsumerConfiguration configuration, + boolean autoCommit) { + RPCHook rpcHook = null; + if (configuration.isAclEnable()) { + rpcHook = new AclClientRPCHook(new SessionCredentials(configuration.getAccessKey(), configuration.getSecretKey())); + } + DefaultLitePullConsumer consumer = new DefaultLitePullConsumer(rpcHook); + consumer.setNamesrvAddr(configuration.getNamesrvAddr()); + String uniqueName = Thread.currentThread().getName() + "-" + System.currentTimeMillis() % 1000; + consumer.setInstanceName(uniqueName); + consumer.setUnitName(uniqueName); + consumer.setAutoCommit(autoCommit); + if (StringUtils.isNotEmpty(configuration.getGroupId())) { + consumer.setConsumerGroup(configuration.getGroupId()); + } + consumer.setLanguage(LanguageCode.JAVA); + return consumer; + } + + /** + * Created when the topic is not exist + * + * @param configuration + * @param config + */ + public static void maybeCreateTopic(RocketMqBaseConfiguration configuration, TopicConfig config) { + log.info("Try to create topic: {}!", config.getTopicName()); + DefaultMQAdminExt defaultMQAdminExt = null; + try { + defaultMQAdminExt = startMQAdminTool(configuration); + if (existTopicRoute(defaultMQAdminExt, config.getTopicName())) { + log.info("Topic [{}] exist!", config.getTopicName()); + // topic exist + return; + } + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Set clusterNameSet = clusterAddrTable.keySet(); + for (String clusterName : clusterNameSet) { + Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + defaultMQAdminExt.createAndUpdateTopicConfig(addr, config); + } + } + } catch (Exception e) { + throw new RuntimeException("Create topic [" + config.getTopicName() + "] failed", e); + } finally { + if (defaultMQAdminExt != null) { + defaultMQAdminExt.shutdown(); + } + } + } + + /** + * create topic + * + * @param configuration + * @param config + */ + public static void createTopic(RocketMqBaseConfiguration configuration, TopicConfig config) { + DefaultMQAdminExt defaultMQAdminExt = null; + try { + defaultMQAdminExt = startMQAdminTool(configuration); + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Set clusterNameSet = clusterAddrTable.keySet(); + for (String clusterName : clusterNameSet) { + Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + defaultMQAdminExt.createAndUpdateTopicConfig(addr, config); + } + } + } catch (Exception e) { + throw new RuntimeException("Create topic [" + config.getTopicName() + "] failed", e); + } finally { + if (defaultMQAdminExt != null) { + defaultMQAdminExt.shutdown(); + } + } + } + + public static Set fetchAllConsumerGroup(RocketMqBaseConfiguration configuration) { + Set consumerGroupSet = Sets.newHashSet(); + DefaultMQAdminExt defaultMQAdminExt = null; + try { + defaultMQAdminExt = startMQAdminTool(configuration); + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + for (BrokerData brokerData : clusterInfo.getBrokerAddrTable().values()) { + SubscriptionGroupWrapper subscriptionGroupWrapper = defaultMQAdminExt.getAllSubscriptionGroup(brokerData.selectBrokerAddr(), 3000L); + consumerGroupSet.addAll(subscriptionGroupWrapper.getSubscriptionGroupTable().keySet()); + } + } catch (Exception e) { + throw new RuntimeException("Fetch all topic failed", e); + } finally { + if (defaultMQAdminExt != null) { + defaultMQAdminExt.shutdown(); + } + } + return consumerGroupSet; + } + + public static String createGroup(RocketMqBaseConfiguration configuration, String group) { + DefaultMQAdminExt defaultMQAdminExt = null; + try { + defaultMQAdminExt = startMQAdminTool(configuration); + SubscriptionGroupConfig initConfig = new SubscriptionGroupConfig(); + initConfig.setGroupName(group); + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Set clusterNameSet = clusterAddrTable.keySet(); + for (String clusterName : clusterNameSet) { + Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, initConfig); + } + } + } catch (Exception e) { + throw new RuntimeException("Create group [" + group + "] failed", e); + } finally { + if (defaultMQAdminExt != null) { + defaultMQAdminExt.shutdown(); + } + } + return group; + } + + /** + * Get topic offsets + */ + public static Map> offsetTopics( + RocketMqBaseConfiguration config, List topics) { + Map> offsets = Maps.newHashMap(); + DefaultMQAdminExt adminClient = null; + try { + adminClient = startMQAdminTool(config); + for (String topic : topics) { + TopicStatsTable topicStatsTable = examineTopicStats(adminClient, topic); + offsets.put(topic, topicStatsTable.getOffsetTable()); + } + return offsets; + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + if (adminClient != null) { + adminClient.shutdown(); + } + } + } + + /** + * Flat topics offsets + */ + public static Map flatOffsetTopics( + RocketMqBaseConfiguration config, List topics) { + Map messageQueueTopicOffsets = Maps.newHashMap(); + offsetTopics(config, topics).values() + .forEach( + offsetTopic -> { + messageQueueTopicOffsets.putAll(offsetTopic); + }); + return messageQueueTopicOffsets; + } + + /** + * Search offsets by timestamp + */ + public static Map searchOffsetsByTimestamp( + RocketMqBaseConfiguration config, + Collection messageQueues, + Long timestamp) { + Map offsets = Maps.newHashMap(); + DefaultMQAdminExt adminClient = null; + try { + adminClient = startMQAdminTool(config); + for (MessageQueue messageQueue : messageQueues) { + long offset = adminClient.searchOffset(messageQueue, timestamp); + offsets.put(messageQueue, offset); + } + return offsets; + } catch (MQClientException e) { + throw new RuntimeException(e); + } finally { + if (adminClient != null) { + adminClient.shutdown(); + } + } + } + + /** + * Get consumer group offset + */ + public static Map currentOffsets(RocketMqBaseConfiguration config, String groupName, + List topics, Set messageQueues) { + // Get consumer group offset + DefaultMQAdminExt adminClient = null; + try { + adminClient = startMQAdminTool(config); + Map consumerOffsets = Maps.newHashMap(); + for (String topic : topics) { + ConsumeStats consumeStats = examineConsumeStats(adminClient, groupName, topic); + consumerOffsets.putAll(consumeStats.getOffsetTable()); + } + return consumerOffsets.keySet().stream() + .filter(messageQueue -> messageQueues.contains(messageQueue)) + .collect( + Collectors.toMap( + messageQueue -> messageQueue, + messageQueue -> + consumerOffsets.get(messageQueue).getConsumerOffset())); + } catch (Exception e) { + if (e instanceof MQClientException) { + if (((MQClientException) e).getResponseCode() == ResponseCode.TOPIC_NOT_EXIST) { + return Collections.emptyMap(); + } else { + throw new RuntimeException(e); + } + } else { + throw new RuntimeException(e); + } + } finally { + if (adminClient != null) { + adminClient.shutdown(); + } + } + } + + private static DefaultMQAdminExt startMQAdminTool( + RocketMqBaseConfiguration configuration) throws MQClientException { + RPCHook rpcHook = null; + if (configuration.isAclEnable()) { + rpcHook = new AclClientRPCHook(new SessionCredentials(configuration.getAccessKey(), configuration.getSecretKey())); + } + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setNamesrvAddr(configuration.getNamesrvAddr()); + defaultMQAdminExt.setAdminExtGroup(configuration.getGroupId()); + defaultMQAdminExt.setInstanceName(createUniqInstance(configuration.getNamesrvAddr())); + defaultMQAdminExt.start(); + return defaultMQAdminExt; + } + + public static boolean isTopicExist(RocketMqBaseConfiguration connectConfig, String topic) { + DefaultMQAdminExt defaultMQAdminExt = null; + try { + defaultMQAdminExt = startMQAdminTool(connectConfig); + return existTopicRoute(defaultMQAdminExt, topic); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + if (defaultMQAdminExt != null) { + defaultMQAdminExt.shutdown(); + } + } + } + + private static boolean existTopicRoute(DefaultMQAdminExt defaultMQAdminExt, String topic) { + boolean foundTopicRouteInfo = false; + try { + TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); + if (topicRouteData != null) { + foundTopicRouteInfo = true; + } + } catch (Exception e) { + if (e instanceof MQClientException) { + if (((MQClientException) e).getResponseCode() == ResponseCode.TOPIC_NOT_EXIST) { + foundTopicRouteInfo = false; + } else { + throw new RuntimeException("Get topic route info failed", e); + } + } else { + throw new RuntimeException("Get topic route info failed", e); + } + } + return foundTopicRouteInfo; + } + + /** + * Compatible with 4.9.4 and earlier + * + * @param adminClient + * @param topic + * @return + */ + private static TopicStatsTable examineTopicStats(DefaultMQAdminExt adminClient, String topic) { + try { + return adminClient.examineTopicStats(topic); + } catch (MQBrokerException e) { + // Compatible with 4.9.4 and earlier + if (e.getResponseCode() == ResponseCode.REQUEST_CODE_NOT_SUPPORTED) { + try { + log.warn("Examine topic stats failure , the server version is less than 5.1.0, and downward compatibility begins, {}", e.getErrorMessage()); + return overrideExamineTopicStats(adminClient, topic); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } else { + throw new RuntimeException(e); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + /** + * examineConsumeStats + * Compatible with 4.9.4 and earlier + * + * @param adminClient + * @param topic + * @return + */ + private static ConsumeStats examineConsumeStats(DefaultMQAdminExt adminClient, String groupName, String topic) { + try { + return adminClient.examineConsumeStats(groupName, topic); + } catch (MQBrokerException e) { + // Compatible with 4.9.4 and earlier + if (e.getResponseCode() == ResponseCode.REQUEST_CODE_NOT_SUPPORTED) { + try { + log.warn("Examine consume stats failure, the server version is less than 5.1.0, and downward compatibility begins {}", e.getErrorMessage()); + return overrideExamineConsumeStats(adminClient, groupName, topic); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } else { + throw new RuntimeException(e); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + /** + * Compatible with version 4.9.4 + * + * @param adminClient + * @param topic + * @return + * @throws RemotingException + * @throws InterruptedException + * @throws MQClientException + * @throws MQBrokerException + */ + private static TopicStatsTable overrideExamineTopicStats(DefaultMQAdminExt adminClient, + String topic) throws RemotingException, InterruptedException, MQClientException, MQBrokerException { + TopicRouteData topicRouteData = adminClient.examineTopicRouteInfo(topic); + TopicStatsTable topicStatsTable = new TopicStatsTable(); + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + TopicStatsTable tst = adminClient + .getDefaultMQAdminExtImpl() + .getMqClientInstance() + .getMQClientAPIImpl() + .getTopicStatsInfo(addr, topic, 5000); + topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable()); + } + } + return topicStatsTable; + } + + /** + * Compatible with version 4.9.4 + * + * @param adminExt + * @param groupName + * @param topic + * @return + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + * @throws MQBrokerException + */ + private static ConsumeStats overrideExamineConsumeStats(DefaultMQAdminExt adminExt, String groupName, + String topic) throws MQClientException, RemotingException, InterruptedException, MQBrokerException { + TopicRouteData topicRouteData = null; + List routeTopics = new ArrayList<>(); + routeTopics.add(MixAll.getRetryTopic(groupName)); + if (topic != null) { + routeTopics.add(topic); + routeTopics.add(KeyBuilder.buildPopRetryTopic(topic, groupName)); + } + for (int i = 0; i < routeTopics.size(); i++) { + try { + topicRouteData = adminExt.getDefaultMQAdminExtImpl().examineTopicRouteInfo(routeTopics.get(i)); + if (topicRouteData != null) { + break; + } + } catch (Throwable e) { + if (i == routeTopics.size() - 1) { + throw e; + } + } + } + ConsumeStats result = new ConsumeStats(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + ConsumeStats consumeStats = adminExt.getDefaultMQAdminExtImpl().getMqClientInstance().getMQClientAPIImpl().getConsumeStats(addr, groupName, topic, 5000 * 3); + result.getOffsetTable().putAll(consumeStats.getOffsetTable()); + double value = result.getConsumeTps() + consumeStats.getConsumeTps(); + result.setConsumeTps(value); + } + } + + Set topics = Sets.newHashSet(); + for (MessageQueue messageQueue : result.getOffsetTable().keySet()) { + topics.add(messageQueue.getTopic()); + } + + ConsumeStats staticResult = new ConsumeStats(); + staticResult.setConsumeTps(result.getConsumeTps()); + + for (String currentTopic : topics) { + TopicRouteData currentRoute = adminExt.getDefaultMQAdminExtImpl().examineTopicRouteInfo(currentTopic); + if (currentRoute.getTopicQueueMappingByBroker() == null + || currentRoute.getTopicQueueMappingByBroker().isEmpty()) { + //normal topic + for (Map.Entry entry : result.getOffsetTable().entrySet()) { + if (entry.getKey().getTopic().equals(currentTopic)) { + staticResult.getOffsetTable().put(entry.getKey(), entry.getValue()); + } + } + } + } + + if (staticResult.getOffsetTable().isEmpty()) { + throw new MQClientException(ResponseCode.CONSUMER_NOT_ONLINE, "Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message"); + } + return staticResult; + } + +} diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/cache/Cache.java b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/cache/Cache.java similarity index 95% rename from rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/cache/Cache.java rename to rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/cache/Cache.java index ebf90522f..aa6030812 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/cache/Cache.java +++ b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/cache/Cache.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.rocketmq.connect.runtime.common.cache; +package org.apache.rocketmq.connect.common.cache; /** * cache diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/cache/LRUCache.java b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/cache/LRUCache.java similarity index 97% rename from rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/cache/LRUCache.java rename to rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/cache/LRUCache.java index 5a32104b1..737725199 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/cache/LRUCache.java +++ b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/cache/LRUCache.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.rocketmq.connect.runtime.common.cache; +package org.apache.rocketmq.connect.common.cache; import java.util.Collections; import java.util.LinkedHashMap; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/LoggerName.java b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/constant/LoggerName.java similarity index 95% rename from rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/LoggerName.java rename to rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/constant/LoggerName.java index 2b62956a5..988aae476 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/common/LoggerName.java +++ b/rocketmq-connect-common/src/main/java/org/apache/rocketmq/connect/common/constant/LoggerName.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.rocketmq.connect.runtime.common; +package org.apache.rocketmq.connect.common.constant; /** * Define all the logger name of the runtime. diff --git a/rocketmq-connect-runtime/pom.xml b/rocketmq-connect-runtime/pom.xml index 4ed44af40..0ed0dd69f 100644 --- a/rocketmq-connect-runtime/pom.xml +++ b/rocketmq-connect-runtime/pom.xml @@ -267,5 +267,10 @@ metric-exporter + + org.apache.rocketmq + rocketmq-connect-common + + diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/DistributedConnectStartup.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/DistributedConnectStartup.java index 4a35f30ce..412c9ddcf 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/DistributedConnectStartup.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/DistributedConnectStartup.java @@ -24,7 +24,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.controller.distributed.DistributedConfig; import org.apache.rocketmq.connect.runtime.controller.distributed.DistributedConnectController; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/StandaloneConnectStartup.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/StandaloneConnectStartup.java index 4b8913e3b..30c2e427f 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/StandaloneConnectStartup.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/StandaloneConnectStartup.java @@ -24,7 +24,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.controller.isolation.Plugin; import org.apache.rocketmq.connect.runtime.controller.standalone.StandaloneConfig; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/TransformChain.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/TransformChain.java index 5592add72..bf3c2c08e 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/TransformChain.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/TransformChain.java @@ -26,7 +26,7 @@ import io.openmessaging.internal.DefaultKeyValue; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.controller.isolation.Plugin; import org.apache.rocketmq.connect.runtime.errors.ErrorReporter; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/Worker.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/Worker.java index f6ff52cf7..583b17981 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/Worker.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/Worker.java @@ -50,8 +50,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.config.SinkConnectorConfig; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerDirectTask.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerDirectTask.java index 59641fd20..cdbde44f1 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerDirectTask.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerDirectTask.java @@ -27,8 +27,8 @@ import io.openmessaging.connector.api.data.RecordOffset; import io.openmessaging.connector.api.data.RecordPartition; import io.openmessaging.connector.api.storage.OffsetStorageReader; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.status.WrapperStatusListener; import org.apache.rocketmq.connect.runtime.errors.RetryWithToleranceOperator; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTask.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTask.java index ee135cbdf..5c2777cab 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTask.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTask.java @@ -52,12 +52,12 @@ import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.metrics.stats.Avg; import org.apache.rocketmq.connect.metrics.stats.CumulativeCount; import org.apache.rocketmq.connect.metrics.stats.Max; import org.apache.rocketmq.connect.metrics.stats.Rate; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.config.SinkConnectorConfig; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTaskContext.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTaskContext.java index 726fd9f02..5b71494d1 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTaskContext.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSinkTaskContext.java @@ -28,8 +28,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSourceTask.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSourceTask.java index f379b7fbb..7b4097d74 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSourceTask.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSourceTask.java @@ -50,12 +50,12 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.metrics.stats.Avg; import org.apache.rocketmq.connect.metrics.stats.CumulativeCount; import org.apache.rocketmq.connect.metrics.stats.Max; import org.apache.rocketmq.connect.metrics.stats.Rate; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.config.SourceConnectorConfig; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; @@ -428,9 +428,7 @@ private String maybeCreateAndGetTopic(ConnectRecord record) { if (StringUtils.isBlank(topic)) { throw new ConnectException("source connect lack of topic config"); } - if (!ConnectUtil.isTopicExist(workerConfig, topic)) { - ConnectUtil.createTopic(workerConfig, new TopicConfig(topic)); - } + ConnectUtil.maybeCreateTopic(workerConfig, new TopicConfig(topic)); return topic; } diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/AbstractConnectController.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/AbstractConnectController.java index 8a8797518..98e65b4f1 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/AbstractConnectController.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/AbstractConnectController.java @@ -23,8 +23,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.Worker; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/distributed/DistributedConnectController.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/distributed/DistributedConnectController.java index 3417f6fd7..bd3aa2938 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/distributed/DistributedConnectController.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/distributed/DistributedConnectController.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.connect.runtime.controller.distributed; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.controller.AbstractConnectController; import org.apache.rocketmq.connect.runtime.controller.isolation.Plugin; import org.apache.rocketmq.connect.runtime.service.ClusterManagementService; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/converter/record/json/JsonConverter.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/converter/record/json/JsonConverter.java index d9e825947..9ed801f78 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/converter/record/json/JsonConverter.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/converter/record/json/JsonConverter.java @@ -33,9 +33,9 @@ import io.openmessaging.connector.api.data.logical.Timestamp; import io.openmessaging.connector.api.errors.ConnectException; import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.connect.runtime.common.LoggerName; -import org.apache.rocketmq.connect.runtime.common.cache.Cache; -import org.apache.rocketmq.connect.runtime.common.cache.LRUCache; +import org.apache.rocketmq.connect.common.constant.LoggerName; +import org.apache.rocketmq.connect.common.cache.Cache; +import org.apache.rocketmq.connect.common.cache.LRUCache; import org.apache.rocketmq.connect.runtime.serialization.JsonDeserializer; import org.apache.rocketmq.connect.runtime.serialization.JsonSerializer; import org.slf4j.Logger; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/errors/DeadLetterQueueReporter.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/errors/DeadLetterQueueReporter.java index b0635e16f..9f4fbd1c0 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/errors/DeadLetterQueueReporter.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/errors/DeadLetterQueueReporter.java @@ -117,12 +117,12 @@ public static DeadLetterQueueReporter build(ConnectorTaskId connectorTaskId, if (dlqTopic.isEmpty()) { return null; } - if (!ConnectUtil.isTopicExist(workerConfig, dlqTopic)) { - TopicConfig topicConfig = new TopicConfig(dlqTopic); - topicConfig.setReadQueueNums(deadLetterQueueConfig.dlqTopicReadQueueNums()); - topicConfig.setWriteQueueNums(deadLetterQueueConfig.dlqTopicWriteQueueNums()); - ConnectUtil.createTopic(workerConfig, topicConfig); - } + + TopicConfig topicConfig = new TopicConfig(dlqTopic); + topicConfig.setReadQueueNums(deadLetterQueueConfig.dlqTopicReadQueueNums()); + topicConfig.setWriteQueueNums(deadLetterQueueConfig.dlqTopicWriteQueueNums()); + ConnectUtil.maybeCreateTopic(workerConfig, topicConfig); + DefaultMQProducer dlqProducer = ConnectUtil.initDefaultMQProducer(workerConfig); return new DeadLetterQueueReporter(dlqProducer, sinkConfig, connectorTaskId, errorMetricsGroup); } diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/metrics/ConnectMetrics.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/metrics/ConnectMetrics.java index 4ad0e56a7..ed3efda79 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/metrics/ConnectMetrics.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/metrics/ConnectMetrics.java @@ -19,10 +19,10 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Reporter; import com.codahale.metrics.Slf4jReporter; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.metrics.IReporter; import org.apache.rocketmq.connect.metrics.MetricsReporter; import org.apache.rocketmq.connect.metrics.ScheduledMetricsReporter; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.utils.Utils; import org.slf4j.Logger; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/rest/ConnectorPluginsResource.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/rest/ConnectorPluginsResource.java index 3a1113b56..cb43b37ec 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/rest/ConnectorPluginsResource.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/rest/ConnectorPluginsResource.java @@ -19,7 +19,7 @@ import io.javalin.http.Context; import io.openmessaging.connector.api.component.task.sink.SinkConnector; import io.openmessaging.connector.api.component.task.source.SourceConnector; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.controller.AbstractConnectController; import org.apache.rocketmq.connect.runtime.controller.isolation.PluginType; import org.apache.rocketmq.connect.runtime.controller.isolation.PluginWrapper; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/rest/RestHandler.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/rest/RestHandler.java index b4f6c378f..d4b8db3de 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/rest/RestHandler.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/rest/RestHandler.java @@ -22,8 +22,8 @@ import io.javalin.Javalin; import io.javalin.http.Context; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.connectorwrapper.WorkerConnector; import org.apache.rocketmq.connect.runtime.connectorwrapper.WorkerTask; import org.apache.rocketmq.connect.runtime.controller.AbstractConnectController; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListDeserializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListDeserializer.java index 9b1e6abe8..2305f6909 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListDeserializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListDeserializer.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.connect.runtime.serialization; import com.alibaba.fastjson.JSONArray; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListSerializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListSerializer.java index 7aec653f4..446926589 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListSerializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListSerializer.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.connect.runtime.serialization; import com.alibaba.fastjson.JSON; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/ConnectKeyValueDeserializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/ConnectKeyValueDeserializer.java index 353fd877b..c5e5d86f2 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/ConnectKeyValueDeserializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/ConnectKeyValueDeserializer.java @@ -18,8 +18,8 @@ package org.apache.rocketmq.connect.runtime.serialization.store; import com.alibaba.fastjson.JSON; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Deserializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/ConnectKeyValueSerializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/ConnectKeyValueSerializer.java index 7042f302e..0482172c2 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/ConnectKeyValueSerializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/ConnectKeyValueSerializer.java @@ -18,8 +18,8 @@ package org.apache.rocketmq.connect.runtime.serialization.store; import com.alibaba.fastjson.JSON; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetDeserializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetDeserializer.java index ce09f518a..373c0455f 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetDeserializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetDeserializer.java @@ -19,7 +19,7 @@ import com.alibaba.fastjson.JSON; import io.openmessaging.connector.api.data.RecordOffset; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Deserializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetSerializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetSerializer.java index 8b6a60c1e..994e7e229 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetSerializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetSerializer.java @@ -19,7 +19,7 @@ import com.alibaba.fastjson.JSON; import io.openmessaging.connector.api.data.RecordOffset; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionDeserializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionDeserializer.java index 7837eea7f..b3a763439 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionDeserializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionDeserializer.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.connect.runtime.serialization.store; import com.alibaba.fastjson.JSON; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Deserializer; import org.apache.rocketmq.connect.runtime.store.ExtendRecordPartition; import org.slf4j.Logger; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionSerializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionSerializer.java index e80c84ca6..a28741d63 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionSerializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionSerializer.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.connect.runtime.serialization.store; import com.alibaba.fastjson.JSON; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Serializer; import org.apache.rocketmq.connect.runtime.store.ExtendRecordPartition; import org.slf4j.Logger; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPositionMapDeserializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPositionMapDeserializer.java index e2f88364b..80c59a28b 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPositionMapDeserializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPositionMapDeserializer.java @@ -19,7 +19,7 @@ import com.alibaba.fastjson.JSON; import io.openmessaging.connector.api.data.RecordOffset; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Deserializer; import org.apache.rocketmq.connect.runtime.store.ExtendRecordPartition; import org.slf4j.Logger; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPositionMapSerializer.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPositionMapSerializer.java index 030bf19d9..e2e9624c1 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPositionMapSerializer.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPositionMapSerializer.java @@ -19,7 +19,7 @@ import com.alibaba.fastjson.JSON; import io.openmessaging.connector.api.data.RecordOffset; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Serializer; import org.apache.rocketmq.connect.runtime.store.ExtendRecordPartition; import org.slf4j.Logger; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractConfigManagementService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractConfigManagementService.java index 8dbf48384..3e9505073 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractConfigManagementService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractConfigManagementService.java @@ -19,8 +19,6 @@ import io.openmessaging.KeyValue; import io.openmessaging.connector.api.component.connector.Connector; -import org.apache.commons.collections.CollectionUtils; -import org.apache.rocketmq.connect.runtime.common.ConfigException; import io.openmessaging.connector.api.component.task.sink.SinkConnector; import io.openmessaging.connector.api.component.task.source.SourceConnector; import io.openmessaging.connector.api.data.RecordConverter; @@ -29,9 +27,19 @@ import io.openmessaging.connector.api.data.SchemaBuilder; import io.openmessaging.connector.api.data.Struct; import io.openmessaging.connector.api.errors.ConnectException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.connect.common.constant.LoggerName; +import org.apache.rocketmq.connect.runtime.common.ConfigException; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.config.SinkConnectorConfig; import org.apache.rocketmq.connect.runtime.config.SourceConnectorConfig; @@ -49,15 +57,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - import static org.apache.rocketmq.connect.runtime.config.ConnectorConfig.CONNECTOR_CLASS; /** @@ -72,53 +71,52 @@ public abstract class AbstractConfigManagementService implements ConfigManagemen public static final String DELETE_CONNECTOR_PREFIX = "delete-"; protected static final String FIELD_STATE = "state"; protected static final String FIELD_EPOCH = "epoch"; - protected static final String FIELD_PROPS = "properties"; - protected static final String FIELD_DELETED = "deleted"; /** * delete connector V0 */ @Deprecated public static final Schema CONNECTOR_DELETE_CONFIGURATION_V0 = SchemaBuilder.struct() - .field(FIELD_EPOCH, SchemaBuilder.int64().build()) - .build(); - - /** - * delete connector V1 - */ - public static final Schema CONNECTOR_DELETE_CONFIGURATION_V1 = SchemaBuilder.struct() - .field(FIELD_EPOCH, SchemaBuilder.int64().build()) - .field(FIELD_DELETED, SchemaBuilder.bool().build()) - .build(); + .field(FIELD_EPOCH, SchemaBuilder.int64().build()) + .build(); /** * connector state */ public static final Schema TARGET_STATE_V0 = SchemaBuilder.struct() - .field(FIELD_STATE, SchemaBuilder.string().build()) - .field(FIELD_EPOCH, SchemaBuilder.int64().build()) - .build(); + .field(FIELD_STATE, SchemaBuilder.string().build()) + .field(FIELD_EPOCH, SchemaBuilder.int64().build()) + .build(); + protected static final String FIELD_PROPS = "properties"; /** * connector configuration */ public static final Schema CONNECTOR_CONFIGURATION_V0 = SchemaBuilder.struct() - .field(FIELD_STATE, SchemaBuilder.string().build()) - .field(FIELD_EPOCH, SchemaBuilder.int64().build()) - .field(FIELD_PROPS, - SchemaBuilder.map( - SchemaBuilder.string().optional().build(), - SchemaBuilder.string().optional().build() - ).build()) - .build(); + .field(FIELD_STATE, SchemaBuilder.string().build()) + .field(FIELD_EPOCH, SchemaBuilder.int64().build()) + .field(FIELD_PROPS, + SchemaBuilder.map( + SchemaBuilder.string().optional().build(), + SchemaBuilder.string().optional().build() + ).build()) + .build(); /** * task configuration */ public static final Schema TASK_CONFIGURATION_V0 = SchemaBuilder.struct() - .field(FIELD_EPOCH, SchemaBuilder.int64().build()) - .field(FIELD_PROPS, - SchemaBuilder.map( - SchemaBuilder.string().build(), - SchemaBuilder.string().optional().build() - ).build()) - .build(); + .field(FIELD_EPOCH, SchemaBuilder.int64().build()) + .field(FIELD_PROPS, + SchemaBuilder.map( + SchemaBuilder.string().build(), + SchemaBuilder.string().optional().build() + ).build()) + .build(); + protected static final String FIELD_DELETED = "deleted"; + /** + * delete connector V1 + */ + public static final Schema CONNECTOR_DELETE_CONFIGURATION_V1 = SchemaBuilder.struct() + .field(FIELD_EPOCH, SchemaBuilder.int64().build()) + .field(FIELD_DELETED, SchemaBuilder.bool().build()) + .build(); private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_RUNTIME); protected final String configManagePrefix = "ConfigManage"; /** @@ -172,7 +170,6 @@ public void initialize(WorkerConfig workerConfig, RecordConverter converter, Plu this.dataSynchronizer = initializationDataSynchronizer(workerConfig); } - @Override public boolean enabledCompactTopic() { return false; @@ -306,7 +303,6 @@ protected void putTaskConfigs(String connectorName, List config taskKeyValueStore.put(connectorName, configs); } - @Override public void recomputeTaskConfigs(String connectorName, ConnectKeyValue configs) { int maxTask = configs.getInt(ConnectorConfig.MAX_TASK, ConnectorConfig.TASKS_MAX_DEFAULT); @@ -394,7 +390,6 @@ public ClusterConfigState snapshot() { return new ClusterConfigState(connectorTaskCounts, connectorConfigs, connectorTargetStates, connectorTaskConfigs); } - @Override public Plugin getPlugin() { return this.plugin; @@ -418,22 +413,22 @@ public void triggerListener() { } } - // ======= Start receives the config message and transforms the storage ====== protected void process(String key, SchemaAndValue schemaAndValue) { if (key.startsWith(TARGET_STATE_PREFIX)) { // target state listener String connectorName = key.substring(TARGET_STATE_PREFIX.length()); + processTargetStateRecord(connectorName, schemaAndValue); + + } else if (key.startsWith(CONNECTOR_PREFIX)) { + // connector config update + String connectorName = key.substring(CONNECTOR_PREFIX.length()); if (schemaAndValue.schema().equals(CONNECTOR_DELETE_CONFIGURATION_V1)) { processDeleteConnectorRecord(connectorName, schemaAndValue); } else { - processTargetStateRecord(connectorName, schemaAndValue); + processConnectorConfigRecord(connectorName, schemaAndValue); } - } else if (key.startsWith(CONNECTOR_PREFIX)) { - // connector config update - String connectorName = key.substring(CONNECTOR_PREFIX.length()); - processConnectorConfigRecord(connectorName, schemaAndValue); } else if (key.startsWith(TASK_PREFIX)) { // task config update ConnectorTaskId taskId = parseTaskId(key); @@ -468,8 +463,12 @@ private void processDeleteConnectorRecord(String connectorName, SchemaAndValue s // config update if ((Long) epoch > oldConfig.getEpoch()) { // remove - connectorKeyValueStore.remove(connectorName); - taskKeyValueStore.remove(connectorName); + if (connectorKeyValueStore.containsKey(connectorName)) { + connectorKeyValueStore.remove(connectorName); + } + if (taskKeyValueStore.containsKey(connectorName)) { + taskKeyValueStore.remove(connectorName); + } // reblance triggerListener(); } @@ -515,14 +514,14 @@ private void processTargetStateRecord(String connectorName, SchemaAndValue schem if (!(targetState instanceof String)) { // target state log.error("Invalid data for target state for connector '{}': 'state' field should be a String but is {}", - connectorName, className(targetState)); + connectorName, className(targetState)); return; } Object epoch = struct.get(FIELD_EPOCH); if (!(epoch instanceof Long)) { // epoch log.error("Invalid data for epoch for connector '{}': 'epoch' field should be a Long but is {}", - connectorName, className(epoch)); + connectorName, className(epoch)); return; } @@ -550,21 +549,21 @@ private boolean mergeConnectConfig(String connectName, SchemaAndValue schemaAndV if (!(targetState instanceof String)) { // target state log.error("Invalid data for target state for connector '{}': 'state' field should be a String but is {}", - connectName, className(targetState)); + connectName, className(targetState)); return false; } Object epoch = value.get(FIELD_EPOCH); if (!(epoch instanceof Long)) { // epoch log.error("Invalid data for epoch for connector '{}': 'state' field should be a long but is {}", - connectName, className(epoch)); + connectName, className(epoch)); return false; } Object props = value.get(FIELD_PROPS); if (!(props instanceof Map)) { // properties log.error("Invalid data for properties for connector '{}': 'state' field should be a Map but is {}", - connectName, className(props)); + connectName, className(props)); return false; } // new configs diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractPositionManagementService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractPositionManagementService.java index 2929d4ad5..a469b1824 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractPositionManagementService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractPositionManagementService.java @@ -21,7 +21,7 @@ import io.openmessaging.connector.api.data.RecordConverter; import io.openmessaging.connector.api.data.RecordOffset; import io.openmessaging.connector.api.data.SchemaAndValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.store.ExtendRecordPartition; import org.apache.rocketmq.connect.runtime.store.KeyValueStore; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractStateManagementService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractStateManagementService.java index edad9f188..c71d439ca 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractStateManagementService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractStateManagementService.java @@ -29,8 +29,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnAndTaskStatus; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.status.AbstractStatus; import org.apache.rocketmq.connect.runtime.connectorwrapper.status.ConnectorStatus; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImpl.java index 6aeca0eec..480be35e7 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ClusterManagementServiceImpl.java @@ -22,7 +22,7 @@ import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.utils.ConnectUtil; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -69,11 +69,9 @@ private void prepare(WorkerConfig connectConfig) { ConnectUtil.createSubGroup(connectConfig, consumerGroup); } String clusterStoreTopic = connectConfig.getClusterStoreTopic(); - if (!ConnectUtil.isTopicExist(connectConfig, clusterStoreTopic)) { - log.info("try to create cluster store topic: {}!", clusterStoreTopic); - TopicConfig topicConfig = new TopicConfig(clusterStoreTopic, 1, 1, 6); - ConnectUtil.createTopic(connectConfig, topicConfig); - } + TopicConfig topicConfig = new TopicConfig(clusterStoreTopic, 1, 1, 6); + log.info("try to create cluster store topic: {}!", clusterStoreTopic); + ConnectUtil.maybeCreateTopic(connectConfig, topicConfig); } diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/DefaultConnectorContext.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/DefaultConnectorContext.java index 34e3521b7..a739e54f6 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/DefaultConnectorContext.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/DefaultConnectorContext.java @@ -18,8 +18,8 @@ package org.apache.rocketmq.connect.runtime.service; import io.openmessaging.connector.api.component.connector.ConnectorContext; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.connectorwrapper.WorkerConnector; import org.apache.rocketmq.connect.runtime.controller.AbstractConnectController; import org.slf4j.Logger; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/RebalanceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/RebalanceImpl.java index 699c71912..048105ae0 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/RebalanceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/RebalanceImpl.java @@ -18,9 +18,9 @@ package org.apache.rocketmq.connect.runtime.service; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.Worker; import org.apache.rocketmq.connect.runtime.controller.AbstractConnectController; @@ -79,7 +79,7 @@ public void checkClusterStoreTopic() { String clusterStoreTopic = this.connectController.getConnectConfig().getClusterStoreTopic(); log.info("cluster store topic not exist, try to create it!"); TopicConfig topicConfig = new TopicConfig(clusterStoreTopic, 1, 1, 6); - ConnectUtil.createTopic(connectConfig, topicConfig); + ConnectUtil.maybeCreateTopic(connectConfig, topicConfig); } } diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/RebalanceService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/RebalanceService.java index 8620aa994..cc4373226 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/RebalanceService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/RebalanceService.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.connect.runtime.service; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.utils.ServiceThread; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalConfigManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalConfigManagementServiceImpl.java index 2b0d2dc37..35fed262e 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalConfigManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalConfigManagementServiceImpl.java @@ -79,7 +79,7 @@ public void initialize(WorkerConfig workerConfig, RecordConverter converter, Plu public DataSynchronizer initializationDataSynchronizer(WorkerConfig workerConfig) { return new BrokerBasedLog<>(workerConfig, this.topic, - ConnectUtil.createGroupName(configManagePrefix, workerConfig.getWorkerId()), + ConnectUtil.generateGroupName(configManagePrefix, workerConfig.getWorkerId()), new ConfigChangeCallback(), Serdes.serdeFrom(String.class), Serdes.serdeFrom(byte[].class), diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalPositionManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalPositionManagementServiceImpl.java index 157a278c4..676636234 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalPositionManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalPositionManagementServiceImpl.java @@ -60,7 +60,7 @@ public DataSynchronizer initializationDataSynchronizer(WorkerConfig workerConfig return new BrokerBasedLog( workerConfig, super.topic, - ConnectUtil.createGroupName(super.positionManagePrefix, workerConfig.getWorkerId()), + ConnectUtil.generateGroupName(super.positionManagePrefix, workerConfig.getWorkerId()), new PositionChangeCallback(), Serdes.serdeFrom(ByteBuffer.class), Serdes.serdeFrom(ByteBuffer.class), diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalStateManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalStateManagementServiceImpl.java index 5598d3be7..292b2ef89 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalStateManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/local/LocalStateManagementServiceImpl.java @@ -77,7 +77,7 @@ public void initialize(WorkerConfig config, RecordConverter converter) { public DataSynchronizer initializationDataSynchronizer(WorkerConfig config) { return new BrokerBasedLog(config, statusTopic, - ConnectUtil.createGroupName(statusManagePrefix, config.getWorkerId()), + ConnectUtil.generateGroupName(statusManagePrefix, config.getWorkerId()), new StatusChangeCallback(), Serdes.serdeFrom(String.class), Serdes.serdeFrom(byte[].class), diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/FilePositionManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/FilePositionManagementServiceImpl.java index 6d8c682e7..880240533 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/FilePositionManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/FilePositionManagementServiceImpl.java @@ -20,7 +20,7 @@ import io.openmessaging.connector.api.data.RecordOffset; import io.openmessaging.connector.api.errors.ConnectException; import org.apache.rocketmq.common.utils.ThreadUtils; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.serialization.store.RecordOffsetSerde; import org.apache.rocketmq.connect.runtime.serialization.store.RecordPartitionSerde; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/MemoryConfigManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/MemoryConfigManagementServiceImpl.java index 4d0d1b76b..ca052d6a9 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/MemoryConfigManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/MemoryConfigManagementServiceImpl.java @@ -22,8 +22,8 @@ import io.openmessaging.connector.api.component.task.source.SourceConnector; import io.openmessaging.connector.api.data.RecordConverter; import io.openmessaging.connector.api.errors.ConnectException; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.config.SinkConnectorConfig; import org.apache.rocketmq.connect.runtime.config.SourceConnectorConfig; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/MemoryStateManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/MemoryStateManagementServiceImpl.java index 85fa0803e..2f7b738ab 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/MemoryStateManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/MemoryStateManagementServiceImpl.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.connect.runtime.service.memory; import io.openmessaging.connector.api.data.RecordConverter; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.status.ConnectorStatus; import org.apache.rocketmq.connect.runtime.connectorwrapper.status.TaskStatus; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/StandaloneRebalanceService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/StandaloneRebalanceService.java index dbe39e233..0025e5b15 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/StandaloneRebalanceService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/memory/StandaloneRebalanceService.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.connect.runtime.service.memory; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.service.ClusterManagementService; import org.apache.rocketmq.connect.runtime.service.ConfigManagementService; import org.apache.rocketmq.connect.runtime.service.RebalanceImpl; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqConfigManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqConfigManagementServiceImpl.java index 899d7cd13..5ac864c3e 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqConfigManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqConfigManagementServiceImpl.java @@ -53,7 +53,7 @@ public boolean enabledCompactTopic() { public DataSynchronizer initializationDataSynchronizer(WorkerConfig workerConfig) { return new BrokerBasedLog<>(workerConfig, this.topic, - ConnectUtil.createGroupName(configManagePrefix, workerConfig.getWorkerId()), + ConnectUtil.generateGroupName(configManagePrefix, workerConfig.getWorkerId()), new ConfigChangeCallback(), Serdes.serdeFrom(String.class), Serdes.serdeFrom(byte[].class), diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqPositionManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqPositionManagementServiceImpl.java index d4ba1381e..1ba9c27af 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqPositionManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqPositionManagementServiceImpl.java @@ -47,7 +47,7 @@ public DataSynchronizer initializationDataSynchronizer(WorkerConfig workerConfig return new BrokerBasedLog( workerConfig, this.topic, - ConnectUtil.createGroupName(positionManagePrefix, workerConfig.getWorkerId()), + ConnectUtil.generateGroupName(positionManagePrefix, workerConfig.getWorkerId()), new PositionChangeCallback(), Serdes.serdeFrom(ByteBuffer.class), Serdes.serdeFrom(ByteBuffer.class), diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqStateManagementServiceImpl.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqStateManagementServiceImpl.java index b8e685664..12e225c09 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqStateManagementServiceImpl.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/rocketmq/RocketMqStateManagementServiceImpl.java @@ -51,7 +51,7 @@ public void initialize(WorkerConfig config, RecordConverter converter) { public DataSynchronizer initializationDataSynchronizer(WorkerConfig config) { return new BrokerBasedLog(config, statusTopic, - ConnectUtil.createGroupName(statusManagePrefix, config.getWorkerId()), + ConnectUtil.generateGroupName(statusManagePrefix, config.getWorkerId()), new StatusChangeCallback(), Serdes.serdeFrom(String.class), Serdes.serdeFrom(byte[].class), diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/strategy/AllocateConnAndTaskStrategyByConsistentHash.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/strategy/AllocateConnAndTaskStrategyByConsistentHash.java index 11094dab6..5c17abc6c 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/strategy/AllocateConnAndTaskStrategyByConsistentHash.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/strategy/AllocateConnAndTaskStrategyByConsistentHash.java @@ -20,9 +20,9 @@ import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter; import org.apache.rocketmq.common.consistenthash.HashFunction; import org.apache.rocketmq.common.consistenthash.Node; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/strategy/DefaultAllocateConnAndTaskStrategy.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/strategy/DefaultAllocateConnAndTaskStrategy.java index a12128d1e..aab037e28 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/strategy/DefaultAllocateConnAndTaskStrategy.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/strategy/DefaultAllocateConnAndTaskStrategy.java @@ -17,9 +17,9 @@ package org.apache.rocketmq.connect.runtime.service.strategy; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs; import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStats.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStats.java index 205598712..6d17558bb 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStats.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStats.java @@ -17,7 +17,7 @@ */ package org.apache.rocketmq.connect.runtime.stats; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsManager.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsManager.java index e4450749e..2ddb6afc1 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsManager.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsManager.java @@ -29,7 +29,7 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; -import static org.apache.rocketmq.connect.runtime.common.LoggerName.ROCKETMQ_CONNECT_STATS; +import static org.apache.rocketmq.connect.common.constant.LoggerName.ROCKETMQ_CONNECT_STATS; public class ConnectStatsManager { diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java index a30259a8d..619878b7c 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/stats/ConnectStatsService.java @@ -19,7 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.ServiceThread; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import java.text.MessageFormat; import java.util.HashMap; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/store/FileBaseKeyValueStore.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/store/FileBaseKeyValueStore.java index 624f4e00c..9b2249b2f 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/store/FileBaseKeyValueStore.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/store/FileBaseKeyValueStore.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.serialization.Serde; import org.apache.rocketmq.connect.runtime.utils.Base64Util; import org.apache.rocketmq.connect.runtime.utils.FileAndPropertyUtil; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/store/PositionStorageWriter.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/store/PositionStorageWriter.java index e683a1553..9281baa7a 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/store/PositionStorageWriter.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/store/PositionStorageWriter.java @@ -23,7 +23,7 @@ import io.openmessaging.connector.api.data.RecordPartition; import io.openmessaging.connector.api.errors.ConnectException; import io.openmessaging.connector.api.storage.OffsetStorageWriter; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.service.PositionManagementService; import org.apache.rocketmq.connect.runtime.utils.datasync.DataSynchronizerCallback; import org.slf4j.Logger; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java index 158d6ada4..b53002318 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtil.java @@ -17,64 +17,41 @@ package org.apache.rocketmq.connect.runtime.utils; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import io.openmessaging.connector.api.data.RecordOffset; import io.openmessaging.connector.api.data.RecordPartition; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.acl.common.AclClientRPCHook; import org.apache.rocketmq.acl.common.SessionCredentials; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.KeyBuilder; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.utils.NetworkUtil; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.ConsumerConfiguration; +import org.apache.rocketmq.connect.common.ProducerConfiguration; +import org.apache.rocketmq.connect.common.RocketMqBaseConfiguration; +import org.apache.rocketmq.connect.common.RocketMqUtils; import org.apache.rocketmq.connect.runtime.config.ConnectorConfig; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.service.strategy.AllocateConnAndTaskStrategy; import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.LanguageCode; -import org.apache.rocketmq.remoting.protocol.ResponseCode; -import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; -import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; -import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; -import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; -import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; -import org.apache.rocketmq.remoting.protocol.route.BrokerData; -import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; -import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.CommandUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static org.apache.rocketmq.connect.runtime.connectorwrapper.WorkerSinkTask.QUEUE_OFFSET; public class ConnectUtil { - private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_RUNTIME); + public static final String SYS_TASK_CG_PREFIX = "connect-"; - public static String createGroupName(String prefix) { + public static String generateGroupName(String prefix) { StringBuilder sb = new StringBuilder(); sb.append(prefix).append("-"); sb.append(NetworkUtil.getLocalAddress()).append("-"); @@ -83,24 +60,12 @@ public static String createGroupName(String prefix) { return sb.toString().replace(".", "-"); } - public static String createGroupName(String prefix, String postfix) { + public static String generateGroupName(String prefix, String postfix) { return new StringBuilder().append(prefix).append("-").append(postfix).toString(); } - public static String createInstance(String servers) { - String[] serversArray = servers.split(";"); - List serversList = new ArrayList(); - for (String server : serversArray) { - if (!serversList.contains(server)) { - serversList.add(server); - } - } - Collections.sort(serversList); - return String.valueOf(serversList.toString().hashCode()); - } - - public static String createUniqInstance(String prefix) { - return new StringBuffer(prefix).append("-").append(UUID.randomUUID().toString()).toString(); + private static String createUniqInstance(String prefix) { + return prefix + "-" + UUID.randomUUID(); } public static AllocateConnAndTaskStrategy initAllocateConnAndTaskStrategy(WorkerConfig connectConfig) { @@ -111,19 +76,43 @@ public static AllocateConnAndTaskStrategy initAllocateConnAndTaskStrategy(Worker } } + private static RocketMqBaseConfiguration toBaseConfiguration(WorkerConfig connectConfig, String group) { + return RocketMqBaseConfiguration + .builder() + .namesrvAddr(connectConfig.getNamesrvAddr()) + .aclEnable(connectConfig.isAclEnable()) + .accessKey(connectConfig.getAccessKey()) + .secretKey(connectConfig.getSecretKey()) + .groupId(group) + .build(); + } + + private static ConsumerConfiguration toConsumerConfiguration(WorkerConfig connectConfig) { + return ConsumerConfiguration + .consumerBuilder() + .namesrvAddr(connectConfig.getNamesrvAddr()) + .aclEnable(connectConfig.isAclEnable()) + .accessKey(connectConfig.getAccessKey()) + .secretKey(connectConfig.getSecretKey()) + .build(); + } + + private static ProducerConfiguration toProducerConfiguration(WorkerConfig connectConfig, String group) { + return ProducerConfiguration + .producerBuilder() + .namesrvAddr(connectConfig.getNamesrvAddr()) + .aclEnable(connectConfig.isAclEnable()) + .accessKey(connectConfig.getAccessKey()) + .secretKey(connectConfig.getSecretKey()) + .groupId(group) + .maxMessageSize(ConnectorConfig.MAX_MESSAGE_SIZE) + .sendMsgTimeout(connectConfig.getOperationTimeout()) + .build(); + } + public static DefaultMQProducer initDefaultMQProducer(WorkerConfig connectConfig) { - RPCHook rpcHook = null; - if (connectConfig.getAclEnable()) { - rpcHook = new AclClientRPCHook(new SessionCredentials(connectConfig.getAccessKey(), connectConfig.getSecretKey())); - } - DefaultMQProducer producer = new DefaultMQProducer(rpcHook); - producer.setNamesrvAddr(connectConfig.getNamesrvAddr()); - producer.setInstanceName(createUniqInstance(connectConfig.getNamesrvAddr())); - producer.setProducerGroup(connectConfig.getRmqProducerGroup()); - producer.setSendMsgTimeout(connectConfig.getOperationTimeout()); - producer.setMaxMessageSize(ConnectorConfig.MAX_MESSAGE_SIZE); - producer.setLanguage(LanguageCode.JAVA); - return producer; + ProducerConfiguration producerConfiguration = toProducerConfiguration(connectConfig, connectConfig.getRmqProducerGroup()); + return RocketMqUtils.initDefaultMQProducer(producerConfiguration); } public static DefaultMQPullConsumer initDefaultMQPullConsumer(WorkerConfig connectConfig) { @@ -142,130 +131,52 @@ public static DefaultMQPullConsumer initDefaultMQPullConsumer(WorkerConfig conne return consumer; } - public static DefaultMQPushConsumer initDefaultMQPushConsumer(WorkerConfig connectConfig) { - RPCHook rpcHook = null; - if (connectConfig.getAclEnable()) { - rpcHook = new AclClientRPCHook(new SessionCredentials(connectConfig.getAccessKey(), connectConfig.getSecretKey())); - } - DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(rpcHook); - consumer.setNamesrvAddr(connectConfig.getNamesrvAddr()); - consumer.setInstanceName(createUniqInstance(connectConfig.getNamesrvAddr())); - consumer.setConsumerGroup(createGroupName(connectConfig.getRmqConsumerGroup())); - consumer.setMaxReconsumeTimes(connectConfig.getRmqMaxRedeliveryTimes()); - consumer.setConsumeTimeout(connectConfig.getRmqMessageConsumeTimeout()); - consumer.setConsumeThreadMin(connectConfig.getRmqMinConsumeThreadNums()); - consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); - consumer.setLanguage(LanguageCode.JAVA); - return consumer; + public static void maybeCreateTopic(WorkerConfig connectConfig, TopicConfig topicConfig) { + RocketMqUtils.maybeCreateTopic(toBaseConfiguration(connectConfig, connectConfig.getAdminExtGroup()), topicConfig); } - public static DefaultMQAdminExt startMQAdminTool(WorkerConfig connectConfig) throws MQClientException { - RPCHook rpcHook = null; - if (connectConfig.getAclEnable()) { - rpcHook = new AclClientRPCHook(new SessionCredentials(connectConfig.getAccessKey(), connectConfig.getSecretKey())); - } - DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); - defaultMQAdminExt.setNamesrvAddr(connectConfig.getNamesrvAddr()); - defaultMQAdminExt.setAdminExtGroup(connectConfig.getAdminExtGroup()); - defaultMQAdminExt.setInstanceName(ConnectUtil.createUniqInstance(connectConfig.getNamesrvAddr())); - defaultMQAdminExt.start(); - return defaultMQAdminExt; + public static boolean isTopicExist(WorkerConfig connectConfig, String topic) { + return RocketMqUtils.isTopicExist(toBaseConfiguration(connectConfig, connectConfig.getAdminExtGroup()), topic); } - public static void createTopic(WorkerConfig connectConfig, TopicConfig topicConfig) { - DefaultMQAdminExt defaultMQAdminExt = null; - try { - defaultMQAdminExt = startMQAdminTool(connectConfig); - ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); - Set clusterNameSet = clusterAddrTable.keySet(); - for (String clusterName : clusterNameSet) { - Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); - for (String addr : masterSet) { - defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); - } - } - } catch (Exception e) { - throw new RuntimeException("Create topic [" + topicConfig.getTopicName() + "] failed", e); - } finally { - if (defaultMQAdminExt != null) { - defaultMQAdminExt.shutdown(); - } - } + public static Set fetchAllConsumerGroupList(WorkerConfig connectConfig) { + RocketMqBaseConfiguration baseConfiguration = toBaseConfiguration(connectConfig, connectConfig.getAdminExtGroup()); + return RocketMqUtils.fetchAllConsumerGroup(baseConfiguration); } - public static boolean isTopicExist(WorkerConfig connectConfig, String topic) { - DefaultMQAdminExt defaultMQAdminExt = null; - boolean foundTopicRouteInfo = false; - try { - defaultMQAdminExt = startMQAdminTool(connectConfig); - TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); - if (topicRouteData != null) { - foundTopicRouteInfo = true; - } - } catch (Exception e) { - if (e instanceof MQClientException) { - if (((MQClientException) e).getResponseCode() == ResponseCode.TOPIC_NOT_EXIST) { - foundTopicRouteInfo = false; - } else { - throw new RuntimeException("Get topic route info failed", e); - } - } else { - throw new RuntimeException("Get topic route info failed", e); - } - } finally { - if (defaultMQAdminExt != null) { - defaultMQAdminExt.shutdown(); - } - } - return foundTopicRouteInfo; + public static String createSubGroup(WorkerConfig connectConfig, String subGroup) { + RocketMqBaseConfiguration baseConfiguration = toBaseConfiguration(connectConfig, connectConfig.getAdminExtGroup()); + return RocketMqUtils.createGroup(baseConfiguration, subGroup); } - public static Set fetchAllConsumerGroupList(WorkerConfig connectConfig) { - Set consumerGroupSet = Sets.newHashSet(); - DefaultMQAdminExt defaultMQAdminExt = null; - try { - defaultMQAdminExt = startMQAdminTool(connectConfig); - ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - for (BrokerData brokerData : clusterInfo.getBrokerAddrTable().values()) { - SubscriptionGroupWrapper subscriptionGroupWrapper = defaultMQAdminExt.getAllSubscriptionGroup(brokerData.selectBrokerAddr(), 3000L); - consumerGroupSet.addAll(subscriptionGroupWrapper.getSubscriptionGroupTable().keySet()); - } - } catch (Exception e) { - throw new RuntimeException("Fetch all topic failed", e); - } finally { - if (defaultMQAdminExt != null) { - defaultMQAdminExt.shutdown(); - } - } - return consumerGroupSet; + /** + * init default lite pull consumer + * + * @param connectConfig + * @return + * @throws MQClientException + */ + public static DefaultLitePullConsumer initDefaultLitePullConsumer(WorkerConfig connectConfig, boolean autoCommit) { + ConsumerConfiguration consumerConfiguration = toConsumerConfiguration(connectConfig); + return RocketMqUtils.initDefaultLitePullConsumer(consumerConfiguration, autoCommit); } - public static String createSubGroup(WorkerConfig connectConfig, String subGroup) { - DefaultMQAdminExt defaultMQAdminExt = null; - try { - defaultMQAdminExt = startMQAdminTool(connectConfig); - SubscriptionGroupConfig initConfig = new SubscriptionGroupConfig(); - initConfig.setGroupName(subGroup); - ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); - Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); - Set clusterNameSet = clusterAddrTable.keySet(); - for (String clusterName : clusterNameSet) { - Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); - for (String addr : masterSet) { - defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, initConfig); - } - } - } catch (Exception e) { - throw new RuntimeException("create subGroup: " + subGroup + " failed", e); - } finally { - if (defaultMQAdminExt != null) { - defaultMQAdminExt.shutdown(); - } - } - return subGroup; + /** + * Get topic offsets + */ + public static Map> offsetTopics(WorkerConfig config, List topics) { + RocketMqBaseConfiguration baseConfiguration = toBaseConfiguration(config, config.getAdminExtGroup()); + return RocketMqUtils.offsetTopics(baseConfiguration, topics); } + /** + * Get consumer group offset + */ + public static Map currentOffsets(WorkerConfig config, String groupName, List topics, + Set messageQueues) { + RocketMqBaseConfiguration baseConfiguration = toBaseConfiguration(config, config.getAdminExtGroup()); + return RocketMqUtils.currentOffsets(baseConfiguration, groupName, topics, messageQueues); + } public static RecordPartition convertToRecordPartition(MessageQueue messageQueue) { Map map = new HashMap<>(); @@ -319,247 +230,4 @@ public static RecordPartition convertToRecordPartition(String topic, String brok return recordPartition; } - /** - * init default lite pull consumer - * - * @param connectConfig - * @return - * @throws MQClientException - */ - public static DefaultLitePullConsumer initDefaultLitePullConsumer(WorkerConfig connectConfig, boolean autoCommit) { - DefaultLitePullConsumer consumer = null; - if (Objects.isNull(consumer)) { - if (StringUtils.isBlank(connectConfig.getAccessKey()) && StringUtils.isBlank(connectConfig.getSecretKey())) { - consumer = new DefaultLitePullConsumer(); - } else { - consumer = new DefaultLitePullConsumer(getAclRPCHook(connectConfig.getAccessKey(), connectConfig.getSecretKey())); - } - } - consumer.setNamesrvAddr(connectConfig.getNamesrvAddr()); - String uniqueName = Thread.currentThread().getName() + "-" + System.currentTimeMillis() % 1000; - consumer.setInstanceName(uniqueName); - consumer.setUnitName(uniqueName); - consumer.setAutoCommit(autoCommit); - return consumer; - } - - private static RPCHook getAclRPCHook(String accessKey, String secretKey) { - return new AclClientRPCHook(new SessionCredentials(accessKey, secretKey)); - } - - /** - * Get topic offsets - */ - public static Map> offsetTopics( - WorkerConfig config, List topics) { - Map> offsets = Maps.newConcurrentMap(); - DefaultMQAdminExt adminClient = null; - try { - adminClient = startMQAdminTool(config); - for (String topic : topics) { - TopicStatsTable topicStatsTable = examineTopicStats(adminClient, topic); - offsets.put(topic, topicStatsTable.getOffsetTable()); - } - return offsets; - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - if (adminClient != null) { - adminClient.shutdown(); - } - } - } - - - /** Get consumer group offset */ - public static Map currentOffsets(WorkerConfig config, String groupName, List topics, Set messageQueues) { - // Get consumer group offset - DefaultMQAdminExt adminClient = null; - try { - adminClient = startMQAdminTool(config); - Map consumerOffsets = Maps.newConcurrentMap(); - for (String topic : topics) { - ConsumeStats consumeStats = examineConsumeStats(adminClient, groupName, topic); - consumerOffsets.putAll(consumeStats.getOffsetTable()); - } - return consumerOffsets.keySet().stream() - .filter(messageQueue -> messageQueues.contains(messageQueue)) - .collect( - Collectors.toMap( - messageQueue -> messageQueue, - messageQueue -> - consumerOffsets.get(messageQueue).getConsumerOffset())); - } catch (MQClientException e) { - if (e instanceof MQClientException) { - if (e.getResponseCode() == ResponseCode.TOPIC_NOT_EXIST) { - return Collections.emptyMap(); - } else { - throw new RuntimeException(e); - } - } else { - throw new RuntimeException(e); - } - } finally { - if (adminClient != null) { - adminClient.shutdown(); - } - } - } - - /** - * Compatible with 4.9.4 and earlier - * - * @param adminClient - * @param topic - * @return - */ - private static TopicStatsTable examineTopicStats(DefaultMQAdminExt adminClient, String topic) { - try { - return adminClient.examineTopicStats(topic); - } catch (MQBrokerException e) { - // Compatible with 4.9.4 and earlier - if (e.getResponseCode() == ResponseCode.REQUEST_CODE_NOT_SUPPORTED) { - try { - log.warn("Examine topic stats failure , the server version is less than 5.1.0, and downward compatibility begins, {}", e.getErrorMessage()); - return overrideExamineTopicStats(adminClient, topic); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } else { - throw new RuntimeException(e); - } - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - /** - * examineConsumeStats - * Compatible with 4.9.4 and earlier - * - * @param adminClient - * @param topic - * @return - */ - private static ConsumeStats examineConsumeStats(DefaultMQAdminExt adminClient, String groupName, String topic) { - try { - return adminClient.examineConsumeStats(groupName, topic); - } catch (MQBrokerException e) { - // Compatible with 4.9.4 and earlier - if (e.getResponseCode() == ResponseCode.REQUEST_CODE_NOT_SUPPORTED) { - try { - log.warn("Examine consume stats failure, the server version is less than 5.1.0, and downward compatibility begins {}", e.getErrorMessage()); - return overrideExamineConsumeStats(adminClient, groupName, topic); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } else { - throw new RuntimeException(e); - } - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - /** - * Compatible with version 4.9.4 - * - * @param adminClient - * @param topic - * @return - * @throws RemotingException - * @throws InterruptedException - * @throws MQClientException - * @throws MQBrokerException - */ - private static TopicStatsTable overrideExamineTopicStats(DefaultMQAdminExt adminClient, - String topic) throws RemotingException, InterruptedException, MQClientException, MQBrokerException { - TopicRouteData topicRouteData = adminClient.examineTopicRouteInfo(topic); - TopicStatsTable topicStatsTable = new TopicStatsTable(); - for (BrokerData bd : topicRouteData.getBrokerDatas()) { - String addr = bd.selectBrokerAddr(); - if (addr != null) { - TopicStatsTable tst = adminClient - .getDefaultMQAdminExtImpl() - .getMqClientInstance() - .getMQClientAPIImpl() - .getTopicStatsInfo(addr, topic, 5000); - topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable()); - } - } - return topicStatsTable; - } - - /** - * Compatible with version 4.9.4 - * - * @param adminExt - * @param groupName - * @param topic - * @return - * @throws MQClientException - * @throws RemotingException - * @throws InterruptedException - * @throws MQBrokerException - */ - private static ConsumeStats overrideExamineConsumeStats(DefaultMQAdminExt adminExt, String groupName, - String topic) throws MQClientException, RemotingException, InterruptedException, MQBrokerException { - TopicRouteData topicRouteData = null; - List routeTopics = new ArrayList<>(); - routeTopics.add(MixAll.getRetryTopic(groupName)); - if (topic != null) { - routeTopics.add(topic); - routeTopics.add(KeyBuilder.buildPopRetryTopic(topic, groupName)); - } - for (int i = 0; i < routeTopics.size(); i++) { - try { - topicRouteData = adminExt.getDefaultMQAdminExtImpl().examineTopicRouteInfo(routeTopics.get(i)); - if (topicRouteData != null) { - break; - } - } catch (Throwable e) { - if (i == routeTopics.size() - 1) { - throw e; - } - } - } - ConsumeStats result = new ConsumeStats(); - - for (BrokerData bd : topicRouteData.getBrokerDatas()) { - String addr = bd.selectBrokerAddr(); - if (addr != null) { - ConsumeStats consumeStats = adminExt.getDefaultMQAdminExtImpl().getMqClientInstance().getMQClientAPIImpl().getConsumeStats(addr, groupName, topic, 5000 * 3); - result.getOffsetTable().putAll(consumeStats.getOffsetTable()); - double value = result.getConsumeTps() + consumeStats.getConsumeTps(); - result.setConsumeTps(value); - } - } - - Set topics = Sets.newHashSet(); - for (MessageQueue messageQueue : result.getOffsetTable().keySet()) { - topics.add(messageQueue.getTopic()); - } - - ConsumeStats staticResult = new ConsumeStats(); - staticResult.setConsumeTps(result.getConsumeTps()); - - for (String currentTopic : topics) { - TopicRouteData currentRoute = adminExt.getDefaultMQAdminExtImpl().examineTopicRouteInfo(currentTopic); - if (currentRoute.getTopicQueueMappingByBroker() == null - || currentRoute.getTopicQueueMappingByBroker().isEmpty()) { - //normal topic - for (Map.Entry entry : result.getOffsetTable().entrySet()) { - if (entry.getKey().getTopic().equals(currentTopic)) { - staticResult.getOffsetTable().put(entry.getKey(), entry.getValue()); - } - } - } - } - - if (staticResult.getOffsetTable().isEmpty()) { - throw new MQClientException(ResponseCode.CONSUMER_NOT_ONLINE, "Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message"); - } - return staticResult; - } - } diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ServiceThread.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ServiceThread.java index 3dacf92be..1c749f633 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ServiceThread.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/ServiceThread.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.connect.runtime.utils; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLog.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLog.java index 01eda1526..f97e83044 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLog.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/BrokerBasedLog.java @@ -39,7 +39,7 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.connect.runtime.common.LoggerName; +import org.apache.rocketmq.connect.common.constant.LoggerName; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.serialization.Serde; import org.apache.rocketmq.connect.runtime.utils.Base64Util; @@ -135,16 +135,13 @@ private void prepare() { log.info("Try to create group: {}!", groupName); ConnectUtil.createSubGroup(workerConfig, groupName); } - if (!ConnectUtil.isTopicExist(workerConfig, topicName)) { - log.info("Try to create store topic: {}!", topicName); - TopicConfig topicConfig = new TopicConfig(topicName, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE); - if (enabledCompactTopic) { - Map attributes = Maps.newConcurrentMap(); - attributes.put("+cleanup.policy", "COMPACTION"); - topicConfig.setAttributes(attributes); - } - ConnectUtil.createTopic(workerConfig, topicConfig); + TopicConfig topicConfig = new TopicConfig(topicName, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE); + if (enabledCompactTopic) { + Map attributes = Maps.newConcurrentMap(); + attributes.put("+cleanup.policy", "COMPACTION"); + topicConfig.setAttributes(attributes); } + ConnectUtil.maybeCreateTopic(workerConfig, topicConfig); } private void initializationAndStartConsumer(WorkerConfig workerConfig, String groupName) { diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/common/cache/LRUCacheTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/common/cache/LRUCacheTest.java index 00a45ca08..a27189431 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/common/cache/LRUCacheTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/common/cache/LRUCacheTest.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.connect.runtime.common.cache; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.connect.common.cache.LRUCache; import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Before; diff --git a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtilTest.java b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtilTest.java index 25bd9b6e2..b1bd48bea 100644 --- a/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtilTest.java +++ b/rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/utils/ConnectUtilTest.java @@ -19,17 +19,12 @@ import io.openmessaging.connector.api.data.RecordOffset; import io.openmessaging.connector.api.data.RecordPartition; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; import org.apache.rocketmq.connect.runtime.config.WorkerConfig; import org.apache.rocketmq.connect.runtime.connectorwrapper.NameServerMocker; import org.apache.rocketmq.connect.runtime.connectorwrapper.ServerResponseMocker; @@ -69,29 +64,13 @@ public void after() { @Test public void createGroupNameTest() { - final String groupName1 = ConnectUtil.createGroupName(TEST_GROUP); + final String groupName1 = ConnectUtil.generateGroupName(TEST_GROUP); Assert.assertTrue(groupName1.startsWith(TEST_GROUP)); - final String groupName2 = ConnectUtil.createGroupName(TEST_GROUP, "group"); + final String groupName2 = ConnectUtil.generateGroupName(TEST_GROUP, "group"); Assert.assertTrue(groupName2.endsWith("group")); } - @Test - public void createInstanceTest() { - String servers = "localhost:9876;localhost:9877"; - final String instance = ConnectUtil.createInstance(servers); - List serverList = new ArrayList<>(); - serverList.add(NAME_SERVER_ADDR); - serverList.add("localhost:9877"); - Assert.assertTrue(instance.equals(String.valueOf(serverList.toString().hashCode()))); - } - - @Test - public void createUniqInstanceTest() { - final String instance = ConnectUtil.createUniqInstance(TEST_GROUP); - Assert.assertTrue(instance.startsWith(TEST_GROUP)); - } - @Test public void initAllocateConnAndTaskStrategyTest(){ WorkerConfig connectConfig = new WorkerConfig(); @@ -110,35 +89,13 @@ public void initDefaultMQProducerTest() { Assert.assertEquals(5000, producer.getPersistConsumerOffsetInterval()); } - @Test - public void initDefaultMQPushConsumerTest() { - WorkerConfig connectConfig = new WorkerConfig(); - connectConfig.setNamesrvAddr(NAME_SERVER_ADDR); - final DefaultMQPushConsumer consumer = ConnectUtil.initDefaultMQPushConsumer(connectConfig); - Assert.assertEquals(NAME_SERVER_ADDR, consumer.getNamesrvAddr()); - Assert.assertEquals(30000, consumer.getPollNameServerInterval()); - Assert.assertEquals(30000, consumer.getHeartbeatBrokerInterval()); - Assert.assertEquals(5000, consumer.getPersistConsumerOffsetInterval()); - } - - @Test - public void startMQAdminToolTest() throws MQClientException { - WorkerConfig connectConfig = new WorkerConfig(); - connectConfig.setNamesrvAddr(NAME_SERVER_ADDR); - final DefaultMQAdminExt defaultMQAdminExt = ConnectUtil.startMQAdminTool(connectConfig); - Assert.assertEquals(NAME_SERVER_ADDR, defaultMQAdminExt.getNamesrvAddr()); - Assert.assertEquals(30000, defaultMQAdminExt.getPollNameServerInterval()); - Assert.assertEquals(30000, defaultMQAdminExt.getHeartbeatBrokerInterval()); - Assert.assertEquals(5000, defaultMQAdminExt.getPersistConsumerOffsetInterval()); - } - @Test public void createTopicTest() { WorkerConfig connectConfig = new WorkerConfig(); connectConfig.setNamesrvAddr(NAME_SERVER_ADDR); TopicConfig topicConfig = new TopicConfig(); topicConfig.setTopicName(TEST_TOPIC); - Assertions.assertThatCode(() -> ConnectUtil.createTopic(connectConfig, topicConfig)).doesNotThrowAnyException(); + Assertions.assertThatCode(() -> ConnectUtil.maybeCreateTopic(connectConfig, topicConfig)).doesNotThrowAnyException(); final boolean exist = ConnectUtil.isTopicExist(connectConfig, TEST_TOPIC); Assert.assertTrue(exist); From da77b2231c18f0326d4f5497b5aadadec0c1d7df Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Wed, 8 Mar 2023 19:53:23 +0800 Subject: [PATCH 7/7] update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fd7044bb2..58579c135 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ rocketmq-connect-sample的主要作用是从源文件中读取数据发送到Roc 2. 64bit JDK 1.8+; 3. Maven 3.2.x或以上版本; 4. 启动 [RocketMQ](https://rocketmq.apache.org/docs/quick-start/); +> 支持 RocketMQ [5.1.0] 版本 5. 创建测试Topic > sh ${ROCKETMQ_HOME}/bin/mqadmin updateTopic -t fileTopic -n localhost:9876 -c DefaultCluster -r 8 -w 8