From cb084216422e3627c8777c69650839403fb8eb76 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Mon, 22 Nov 2021 22:33:02 +0400 Subject: [PATCH 001/154] [th2-2689] adaptation to book/page cradleapi (#145) --- README.md | 9 ++-- build.gradle | 4 +- gradle.properties | 2 +- .../schema/factory/AbstractCommonFactory.java | 50 +++++++++++-------- .../schema/cradle/CradleConfiguration.kt | 16 +++--- .../common/schema/TestJsonConfiguration.kt | 3 +- .../cradle_confidential.json | 3 +- 7 files changed, 47 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 521eea2ef..8fb549a40 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (3.29.2) +# th2 common library (Java) (4.0.0) ## Usage @@ -167,7 +167,6 @@ The `CommonFactory` reads a Cradle configuration from the cradle.json file. * keyspace - the required setting defines the keyspace (top-level database object) in the Cassandra data center. * username - the required setting defines the Cassandra username. The user must have permission to write data using a specified keyspace. * password - the required setting defines the password that will be used for connecting to Cassandra. -* cradleInstanceName - this option defines a special identifier that divides data within one keyspace with infra set as the default value. * cradleMaxEventBatchSize - this option defines the maximum event batch size in bytes with its default value set to 1048576. * cradleMaxMessageBatchSize - this option defines the maximum message batch size in bytes with its default value set to 1048576. * timeout - this option defines connection timeout in milliseconds. If set to 0 or ommited, the default value of 5000 is used. @@ -181,7 +180,6 @@ The `CommonFactory` reads a Cradle configuration from the cradle.json file. "keyspace": "", "username": "", "password": "", - "cradleInstanceName": "", "cradleMaxEventBatchSize": 1048576, "cradleMaxMessageBatchSize": 1048576, "timeout": 5000, @@ -274,6 +272,11 @@ EVENTS METRICS: ## Release notes +### 4.0.0 + ++ Adaptation to books/pages cradleapi 4.0.0 ++ Removed `cradleInstanceName` parameter from `cradle.json` + ### 3.29.2 + Do not publish messages if the whole batch was filtered diff --git a/build.gradle b/build.gradle index 8a2aa7272..1cab7eb37 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '2.20.0' + cradleVersion = '4.0.0-TH2-2150-1411749626-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } @@ -158,7 +158,7 @@ test { dependencies { api platform('com.exactpro.th2:bom:3.0.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'com.exactpro.th2:grpc-common:3.8.0' + api 'com.exactpro.th2:grpc-common:3.8.0-th2-2567-1380894854-SNAPSHOT' implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.1.12' diff --git a/gradle.properties b/gradle.properties index e4b9deab8..a72889179 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.29.2 +release_version=4.0.0 description = 'th2 common library (Java)' diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 33333e10d..b98b7e424 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -17,7 +17,7 @@ import com.exactpro.cradle.CradleManager; import com.exactpro.cradle.cassandra.CassandraCradleManager; -import com.exactpro.cradle.cassandra.connection.CassandraConnection; +import com.exactpro.cradle.cassandra.CassandraStorageSettings; import com.exactpro.cradle.cassandra.connection.CassandraConnectionSettings; import com.exactpro.cradle.utils.CradleStorageException; import com.exactpro.th2.common.event.Event; @@ -98,8 +98,8 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_MAX_EVENT_BATCH_SIZE; -import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_MAX_MESSAGE_BATCH_SIZE; +import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_CONSISTENCY_LEVEL; +import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_TIMEOUT; import static com.exactpro.th2.common.schema.util.ArchiveUtils.getGzipBase64StringDecoder; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; @@ -452,35 +452,43 @@ public CradleManager getCradleManager() { if (manager == null) { try { CradleConfidentialConfiguration confidentialConfiguration = getCradleConfidentialConfiguration(); - CradleNonConfidentialConfiguration nonConfidentialConfiguration = getCradleNonConfidentialConfiguration(); - CassandraConnectionSettings cassandraConnectionSettings = new CassandraConnectionSettings( - confidentialConfiguration.getDataCenter(), confidentialConfiguration.getHost(), confidentialConfiguration.getPort(), - confidentialConfiguration.getKeyspace()); - + confidentialConfiguration.getDataCenter() + ); if (StringUtils.isNotEmpty(confidentialConfiguration.getUsername())) { cassandraConnectionSettings.setUsername(confidentialConfiguration.getUsername()); } - if (StringUtils.isNotEmpty(confidentialConfiguration.getPassword())) { cassandraConnectionSettings.setPassword(confidentialConfiguration.getPassword()); } - if (nonConfidentialConfiguration.getTimeout() > 0) { - cassandraConnectionSettings.setTimeout(nonConfidentialConfiguration.getTimeout()); - } - + CradleNonConfidentialConfiguration nonConfidentialConfiguration = getCradleNonConfidentialConfiguration(); + CassandraStorageSettings cassandraStorageSettings = new CassandraStorageSettings( + null, + nonConfidentialConfiguration.getTimeout() > 0 + ? nonConfidentialConfiguration.getTimeout() + : DEFAULT_TIMEOUT, + DEFAULT_CONSISTENCY_LEVEL, + DEFAULT_CONSISTENCY_LEVEL + ); if (nonConfidentialConfiguration.getPageSize() > 0) { - cassandraConnectionSettings.setResultPageSize(nonConfidentialConfiguration.getPageSize()); + cassandraStorageSettings.setResultPageSize(nonConfidentialConfiguration.getPageSize()); + } + if (nonConfidentialConfiguration.getCradleMaxMessageBatchSize() > 0) { + cassandraStorageSettings.setMaxMessageBatchSize(nonConfidentialConfiguration.getCradleMaxMessageBatchSize()); + } + if (nonConfidentialConfiguration.getCradleMaxEventBatchSize() > 0) { + cassandraStorageSettings.setMaxTestEventBatchSize(nonConfidentialConfiguration.getCradleMaxEventBatchSize()); } - manager = new CassandraCradleManager(new CassandraConnection(cassandraConnectionSettings)); - manager.init(defaultIfBlank(confidentialConfiguration.getCradleInstanceName(), DEFAULT_CRADLE_INSTANCE_NAME), true /* FIXME: should be `false` when db manipulations are moved to operator */, - nonConfidentialConfiguration.getCradleMaxMessageBatchSize() > 0 ? nonConfidentialConfiguration.getCradleMaxMessageBatchSize() : DEFAULT_MAX_MESSAGE_BATCH_SIZE, - nonConfidentialConfiguration.getCradleMaxEventBatchSize() > 0 ? nonConfidentialConfiguration.getCradleMaxEventBatchSize() : DEFAULT_MAX_EVENT_BATCH_SIZE); - } catch (CradleStorageException | RuntimeException e) { + manager = new CassandraCradleManager( + cassandraConnectionSettings, + cassandraStorageSettings, + false + ); + } catch (CradleStorageException | RuntimeException | IOException e) { throw new CommonFactoryException("Cannot create Cradle manager", e); } } @@ -732,9 +740,9 @@ public void close() { cradleManager.getAndUpdate(manager -> { if (manager != null) { try { - manager.dispose(); + manager.close(); } catch (Exception e) { - LOGGER.error("Failed to dispose Cradle manager", e); + LOGGER.error("Failed to close Cradle manager", e); } } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt index 575a5d8cf..a0344935b 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt @@ -15,6 +15,7 @@ package com.exactpro.th2.common.schema.cradle +import com.exactpro.cradle.CradleStorage import com.exactpro.cradle.cassandra.CassandraStorageSettings import com.exactpro.th2.common.schema.configuration.Configuration import com.fasterxml.jackson.annotation.JsonProperty @@ -27,12 +28,11 @@ data class CradleConfiguration( var port: Int, var username: String?, var password: String?, - var cradleInstanceName: String?, var timeout: Long, var pageSize: Int, - var cradleMaxEventBatchSize: Long, - var cradleMaxMessageBatchSize: Long -) : Configuration() { + var cradleMaxEventBatchSize: Int, + var cradleMaxMessageBatchSize: Int +) : Configuration() { constructor( cradleConfidentialConfiguration: CradleConfidentialConfiguration, cradleNonConfidentialConfiguration: CradleNonConfidentialConfiguration @@ -43,7 +43,6 @@ data class CradleConfiguration( cradleConfidentialConfiguration.port, cradleConfidentialConfiguration.username, cradleConfidentialConfiguration.password, - cradleConfidentialConfiguration.cradleInstanceName, cradleNonConfidentialConfiguration.timeout, cradleNonConfidentialConfiguration.pageSize, cradleNonConfidentialConfiguration.cradleMaxEventBatchSize, @@ -57,13 +56,12 @@ data class CradleConfidentialConfiguration( @JsonProperty(required = true) var keyspace: String, var port: Int = 0, var username: String? = null, - var password: String? = null, - var cradleInstanceName: String? = null + var password: String? = null ) : Configuration() data class CradleNonConfidentialConfiguration( var timeout: Long = CassandraStorageSettings.DEFAULT_TIMEOUT, var pageSize: Int = 5000, - var cradleMaxEventBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_EVENT_BATCH_SIZE, - var cradleMaxMessageBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_MESSAGE_BATCH_SIZE + var cradleMaxEventBatchSize: Int = CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE, + var cradleMaxMessageBatchSize: Int = CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE ) : Configuration() \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index 5e7bab3e7..09804e382 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -206,8 +206,7 @@ class TestJsonConfiguration { "keyspace", 1234, "user", - "pass", - "instance" + "pass" ) private val CRADLE_NON_CONFIDENTIAL_CONF_JSON = loadConfJson("cradle_non_confidential") diff --git a/src/test/resources/test_json_configurations/cradle_confidential.json b/src/test/resources/test_json_configurations/cradle_confidential.json index e42ae5261..b38a5c6ee 100644 --- a/src/test/resources/test_json_configurations/cradle_confidential.json +++ b/src/test/resources/test_json_configurations/cradle_confidential.json @@ -4,6 +4,5 @@ "keyspace": "keyspace", "port": 1234, "username": "user", - "password": "pass", - "cradleInstanceName": "instance" + "password": "pass" } \ No newline at end of file From bdf5e56a6276c6198c9616949f6382d7e9af3511 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Tue, 23 Nov 2021 13:11:02 +0400 Subject: [PATCH 002/154] [th2-2716] new cradlrapi version (#148) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 1cab7eb37..d0f265a33 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-1411749626-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-1493588338-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 55ef4181d28c429744bfff8f826e25b3f331ab4a Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Wed, 24 Nov 2021 16:54:30 +0400 Subject: [PATCH 003/154] [th2-2469/th2-2567] eventId/messageId builders and pass bookName (#144) --- build.gradle | 2 +- .../box/configuration/BoxConfiguration.java | 15 ++- .../common/schema/event/EventBatchRouter.java | 5 +- .../common/schema/event/EventBatchSender.java | 21 +++- .../schema/factory/AbstractCommonFactory.java | 26 ++++- .../common/schema/factory/CommonFactory.java | 10 +- .../common/schema/message/MessageRouter.java | 4 +- .../impl/rabbitmq/AbstractRabbitSender.java | 5 +- .../th2/common/message/MessageUtils.kt | 19 +++- .../schema/message/MessageRouterContext.kt | 4 +- .../context/DefaultMessageRouterContext.kt | 6 +- .../impl/rabbitmq/AbstractRabbitRouter.kt | 9 +- .../rabbitmq/custom/RabbitCustomRouter.kt | 7 +- .../group/RabbitMessageGroupBatchRouter.kt | 10 +- .../group/RabbitMessageGroupBatchSender.kt | 45 ++++++-- .../builder/MessageEventIdBuildersTest.java | 103 ++++++++++++++++++ .../TestRabbitMessageGroupBatchRouter.kt | 4 +- .../test_message_event_id_builders/box.json | 4 + 18 files changed, 261 insertions(+), 38 deletions(-) create mode 100644 src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java create mode 100644 src/test/resources/test_message_event_id_builders/box.json diff --git a/build.gradle b/build.gradle index d0f265a33..134ea063b 100644 --- a/build.gradle +++ b/build.gradle @@ -158,7 +158,7 @@ test { dependencies { api platform('com.exactpro.th2:bom:3.0.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'com.exactpro.th2:grpc-common:3.8.0-th2-2567-1380894854-SNAPSHOT' + api 'com.exactpro.th2:grpc-common:3.9.0-th2-2150-books-pages-1498965636-SNAPSHOT' implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.1.12' diff --git a/src/main/java/com/exactpro/th2/common/schema/box/configuration/BoxConfiguration.java b/src/main/java/com/exactpro/th2/common/schema/box/configuration/BoxConfiguration.java index 618210e47..678f1cf20 100644 --- a/src/main/java/com/exactpro/th2/common/schema/box/configuration/BoxConfiguration.java +++ b/src/main/java/com/exactpro/th2/common/schema/box/configuration/BoxConfiguration.java @@ -15,14 +15,19 @@ package com.exactpro.th2.common.schema.box.configuration; -import com.fasterxml.jackson.annotation.JsonProperty; import org.jetbrains.annotations.Nullable; +import com.fasterxml.jackson.annotation.JsonProperty; + public class BoxConfiguration { + public static final String DEFAULT_BOOK_NAME = "test_book"; @JsonProperty private String boxName = null; + @JsonProperty + private String bookName = DEFAULT_BOOK_NAME; + @Nullable public String getBoxName() { return boxName; @@ -31,4 +36,12 @@ public String getBoxName() { public void setBoxName(@Nullable String boxName) { this.boxName = boxName; } + + public String getBookName() { + return bookName; + } + + public void setBookName(String bookName) { + this.bookName = bookName; + } } diff --git a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java index c4f25c878..32ab77768 100644 --- a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java @@ -59,12 +59,13 @@ protected Set getRequiredSubscribeAttributes() { @NotNull @Override - protected MessageSender createSender(QueueConfiguration queueConfiguration, @NotNull String pinName) { + protected MessageSender createSender(QueueConfiguration queueConfiguration, @NotNull String pinName, @NotNull String bookName) { return new EventBatchSender( getConnectionManager(), queueConfiguration.getExchange(), queueConfiguration.getRoutingKey(), - pinName + pinName, + bookName ); } diff --git a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSender.java b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSender.java index 661d4ab70..810e5dc8a 100644 --- a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSender.java +++ b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSender.java @@ -20,6 +20,7 @@ import org.jetbrains.annotations.NotNull; +import com.exactpro.th2.common.grpc.Event; import com.exactpro.th2.common.grpc.EventBatch; import com.exactpro.th2.common.message.MessageUtils; import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSender; @@ -27,6 +28,7 @@ import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_PIN_LABEL; import static com.exactpro.th2.common.schema.event.EventBatchRouter.EVENT_TYPE; + import io.prometheus.client.Counter; public class EventBatchSender extends AbstractRabbitSender { @@ -40,9 +42,10 @@ public EventBatchSender( @NotNull ConnectionManager connectionManager, @NotNull String exchangeName, @NotNull String routingKey, - @NotNull String th2Pin + @NotNull String th2Pin, + @NotNull String bookName ) { - super(connectionManager, exchangeName, routingKey, th2Pin, EVENT_TYPE); + super(connectionManager, exchangeName, routingKey, th2Pin, EVENT_TYPE, bookName); } @Override @@ -50,7 +53,19 @@ public void send(EventBatch value) throws IOException { EVENT_PUBLISH_TOTAL .labels(th2Pin) .inc(value.getEventsCount()); - super.send(value); + if (value.getEventsList().stream().anyMatch(event -> event.getId().getBookName().isEmpty())) { + EventBatch.Builder eventBatchBuilder = EventBatch.newBuilder(); + value.getEventsList().forEach(event -> { + Event.Builder eventBuilder = event.toBuilder(); + if (event.getId().getBookName().isEmpty()) { + eventBuilder.getIdBuilder().setBookName(bookName); + } + eventBatchBuilder.addEvents(eventBuilder); + }); + super.send(eventBatchBuilder.build()); + } else { + super.send(value); + } } @Override diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index b98b7e424..28758f831 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -22,8 +22,10 @@ import com.exactpro.cradle.utils.CradleStorageException; import com.exactpro.th2.common.event.Event; import com.exactpro.th2.common.grpc.EventBatch; +import com.exactpro.th2.common.grpc.EventID; import com.exactpro.th2.common.grpc.MessageBatch; import com.exactpro.th2.common.grpc.MessageGroupBatch; +import com.exactpro.th2.common.grpc.MessageID; import com.exactpro.th2.common.grpc.RawMessageBatch; import com.exactpro.th2.common.metrics.CommonMetrics; import com.exactpro.th2.common.metrics.MetricMonitor; @@ -287,7 +289,12 @@ public MessageRouter getEventBatchRouter() { if (router == null) { try { router = eventBatchRouterClass.getConstructor().newInstance(); - router.init(new DefaultMessageRouterContext(getRabbitMqConnectionManager(), MessageRouterMonitor.DEFAULT_MONITOR, getMessageRouterConfiguration())); + router.init(new DefaultMessageRouterContext( + getRabbitMqConnectionManager(), + MessageRouterMonitor.DEFAULT_MONITOR, + getMessageRouterConfiguration(), + getBoxConfiguration() + )); } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { throw new CommonFactoryException("Can not create event batch router", e); } @@ -640,7 +647,12 @@ protected MessageRouterContext getMessageRouterContext() { contextMonitor = new BroadcastMessageRouterMonitor(new LogMessageRouterMonitor(), new EventMessageRouterMonitor(getEventBatchRouter(), rootEventId)); } - return new DefaultMessageRouterContext(getRabbitMqConnectionManager(), contextMonitor, getMessageRouterConfiguration()); + return new DefaultMessageRouterContext( + getRabbitMqConnectionManager(), + contextMonitor, + getMessageRouterConfiguration(), + getBoxConfiguration() + ); } catch (Exception e) { throw new CommonFactoryException("Can not create message router context", e); } @@ -666,6 +678,16 @@ protected ConnectionManager getRabbitMqConnectionManager() { }); } + public MessageID.Builder newMessageIDBuilder() { + return MessageID.newBuilder() + .setBookName(getBoxConfiguration().getBookName()); + } + + public EventID.Builder newEventIDBuilder() { + return EventID.newBuilder() + .setBookName(getBoxConfiguration().getBookName()); + } + @Override public void close() { LOGGER.info("Closing common factory"); diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index edf560142..7084be1b4 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -445,9 +445,6 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam } if (generatedConfigsDirFile.exists()) { - BoxConfiguration box = new BoxConfiguration(); - box.setBoxName(boxName); - settings.setRabbitMQ(writeFile(configPath, RABBIT_MQ_FILE_NAME, rabbitMqData)); settings.setRouterMQ(writeFile(configPath, ROUTER_MQ_FILE_NAME, boxData)); settings.setConnectionManagerSettings(writeFile(configPath, CONNECTION_MANAGER_CONF_FILE_NAME, boxData)); @@ -463,10 +460,13 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam String boxConfig = boxData.get(BOX_FILE_NAME); - if (boxConfig != null) + if (boxConfig != null) { writeFile(boxConfigurationPath, boxConfig); - else + } else { + BoxConfiguration box = new BoxConfiguration(); + box.setBoxName(boxName); writeToJson(boxConfigurationPath, box); + } writeDictionaries(boxName, configPath, dictionaryPath, dictionaries, configMaps.list()); } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index 8a2aef5b2..27113be53 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -16,6 +16,7 @@ package com.exactpro.th2.common.schema.message; import com.exactpro.th2.common.grpc.MessageGroupBatch; +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration; import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration; import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; @@ -41,8 +42,7 @@ public interface MessageRouter extends AutoCloseable { default void init(@NotNull ConnectionManager connectionManager, @NotNull MessageRouterConfiguration configuration) { Objects.requireNonNull(connectionManager, "Connection owner can not be null"); Objects.requireNonNull(configuration, "Configuration cannot be null"); - - init(new DefaultMessageRouterContext(connectionManager, MessageRouterMonitor.DEFAULT_MONITOR, configuration)); + init(new DefaultMessageRouterContext(connectionManager, MessageRouterMonitor.DEFAULT_MONITOR, configuration, new BoxConfiguration())); } default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter groupBatchRouter) { diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSender.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSender.java index 751bc4113..dfa769dce 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSender.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSender.java @@ -51,6 +51,7 @@ public abstract class AbstractRabbitSender implements MessageSender { .register(); protected final String th2Pin; + protected final String bookName; private final AtomicReference routingKey = new AtomicReference<>(); private final AtomicReference exchangeName = new AtomicReference<>(); private final AtomicReference connectionManager = new AtomicReference<>(); @@ -61,13 +62,15 @@ public AbstractRabbitSender( @NotNull String exchangeName, @NotNull String routingKey, @NotNull String th2Pin, - @NotNull String th2Type + @NotNull String th2Type, + @NotNull String bookName ) { this.connectionManager.set(requireNonNull(connectionManager, "Connection can not be null")); this.exchangeName.set(requireNonNull(exchangeName, "Exchange name can not be null")); this.routingKey.set(requireNonNull(routingKey, "Routing key can not be null")); this.th2Pin = requireNonNull(th2Pin, "TH2 pin can not be null"); this.th2Type = requireNonNull(th2Type, "TH2 type can not be null"); + this.bookName = requireNonNull(bookName, "Book name can not be null"); } @Deprecated diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt index 5328995ac..7edc55852 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt @@ -51,6 +51,8 @@ import com.exactpro.th2.common.grpc.Value.KindCase.NULL_VALUE import com.exactpro.th2.common.grpc.Value.KindCase.SIMPLE_VALUE import com.exactpro.th2.common.grpc.ValueFilter import com.exactpro.th2.common.grpc.ValueOrBuilder +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName import com.exactpro.th2.common.value.getBigDecimal import com.exactpro.th2.common.value.getBigInteger import com.exactpro.th2.common.value.getDouble @@ -149,7 +151,14 @@ fun Message.copy(): Message.Builder = Message.newBuilder().setMetadata(metadata) fun Message.Builder.copy(): Message.Builder = Message.newBuilder().setMetadata(metadata).putAllFields(fieldsMap).setParentEventId(parentEventId) -fun Message.Builder.setMetadata(messageType: String? = null, direction: Direction? = null, sessionAlias: String? = null, sequence: Long? = null, timestamp: Instant? = null): Message.Builder = +fun Message.Builder.setMetadata( + messageType: String? = null, + direction: Direction? = null, + sessionAlias: String? = null, + sequence: Long? = null, + timestamp: Instant? = null, + bookName: String = BoxConfiguration.DEFAULT_BOOK_NAME +): Message.Builder = setMetadata(MessageMetadata.newBuilder().also { if (messageType != null) { it.messageType = messageType @@ -166,6 +175,7 @@ fun Message.Builder.setMetadata(messageType: String? = null, direction: Directio if (sequence != null) { this.sequence = sequence } + this.bookName = bookName }.build() } }) @@ -346,6 +356,13 @@ val AnyMessage.sequence: Long else -> error("Message ${shortDebugString(this)} doesn't have message or rawMessage") } +val AnyMessage.bookName: BookName + get() = when { + hasMessage() -> message.metadata.id.bookName + hasRawMessage() -> rawMessage.metadata.id.bookName + else -> error("Message ${shortDebugString(this)} doesn't have message or rawMessage") + } + fun getDebugString(className: String, ids: List): String { val sessionAliasAndDirection = getSessionAliasAndDirection(ids[0]) val sequences = ids.joinToString { it.sequence.toString() } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterContext.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterContext.kt index 915cc3f6f..d62f6da0d 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterContext.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterContext.kt @@ -15,13 +15,13 @@ package com.exactpro.th2.common.schema.message +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager interface MessageRouterContext { - val connectionManager: ConnectionManager val routerMonitor: MessageRouterMonitor val configuration: MessageRouterConfiguration - + val boxConfiguration: BoxConfiguration } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/context/DefaultMessageRouterContext.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/context/DefaultMessageRouterContext.kt index 88915861d..4e19a1f4a 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/context/DefaultMessageRouterContext.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/context/DefaultMessageRouterContext.kt @@ -15,6 +15,7 @@ package com.exactpro.th2.common.schema.message.impl.context +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.message.MessageRouterContext import com.exactpro.th2.common.schema.message.MessageRouterMonitor import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration @@ -23,5 +24,6 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.Connectio class DefaultMessageRouterContext( override val connectionManager: ConnectionManager, override val routerMonitor: MessageRouterMonitor, - override val configuration: MessageRouterConfiguration -) : MessageRouterContext {} \ No newline at end of file + override val configuration: MessageRouterConfiguration, + override val boxConfiguration: BoxConfiguration +) : MessageRouterContext \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index 28ae3136c..c233298c4 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -14,6 +14,7 @@ */ package com.exactpro.th2.common.schema.message.impl.rabbitmq +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.exception.RouterException import com.exactpro.th2.common.schema.filter.strategy.FilterStrategy import com.exactpro.th2.common.schema.message.* @@ -32,6 +33,7 @@ typealias PinName = String typealias PinConfiguration = QueueConfiguration typealias Queue = String typealias RoutingKey = String +typealias BookName = String abstract class AbstractRabbitRouter : MessageRouter { private val _context = AtomicReference() @@ -47,6 +49,9 @@ abstract class AbstractRabbitRouter : MessageRouter { protected val connectionManager: ConnectionManager get() = context.connectionManager + private val boxConfiguration: BoxConfiguration + get() = context.boxConfiguration + private val subscribers = ConcurrentHashMap>() private val senders = ConcurrentHashMap>() @@ -133,7 +138,7 @@ abstract class AbstractRabbitRouter : MessageRouter { protected open fun getRequiredSubscribeAttributes() = REQUIRED_SUBSCRIBE_ATTRIBUTES //TODO: implement common sender - protected abstract fun createSender(pinConfig: PinConfiguration, pinName: PinName): MessageSender + protected abstract fun createSender(pinConfig: PinConfiguration, pinName: PinName, bookName: BookName): MessageSender //TODO: implement common subscriber protected abstract fun createSubscriber(pinConfig: PinConfiguration, pinName: PinName): MessageSubscriber @@ -223,7 +228,7 @@ abstract class AbstractRabbitRouter : MessageRouter { "The $pinName isn't writable, configuration: $pinConfig" } - return@computeIfAbsent createSender(pinConfig, pinName) + return@computeIfAbsent createSender(pinConfig, pinName, boxConfiguration.bookName) } private fun ConcurrentHashMap>.getSubscriber( diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt index 5c6296239..496a03526 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt @@ -22,6 +22,7 @@ import com.exactpro.th2.common.schema.message.QueueAttribute import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSender import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber +import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager @@ -58,12 +59,13 @@ class RabbitCustomRouter( return message } - override fun createSender(pinConfig: PinConfiguration, pinName: PinName): MessageSender { + override fun createSender(pinConfig: PinConfiguration, pinName: PinName, bookName: BookName): MessageSender { return Sender( connectionManager, pinConfig.exchange, pinConfig.routingKey, pinName, + bookName, customTag, converter ) @@ -89,9 +91,10 @@ class RabbitCustomRouter( exchangeName: String, routingKey: String, th2Pin: String, + bookName: BookName, customTag: String, private val converter: MessageConverter - ) : AbstractRabbitSender(connectionManager, exchangeName, routingKey, th2Pin, customTag) { + ) : AbstractRabbitSender(connectionManager, exchangeName, routingKey, th2Pin, customTag, bookName) { override fun valueToBytes(value: T): ByteArray = converter.toByteArray(value) override fun toShortTraceString(value: T): String = converter.toTraceString(value) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt index d1baf2502..42dd9e658 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt @@ -29,6 +29,7 @@ import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitRouter +import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName import com.google.protobuf.Message import com.google.protobuf.TextFormat @@ -66,12 +67,17 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() return if (builder.groupsCount > 0) builder.build() else null } - override fun createSender(pinConfig: QueueConfiguration, pinName: PinName): MessageSender { + override fun createSender( + pinConfig: QueueConfiguration, + pinName: PinName, + bookName: BookName + ): MessageSender { return RabbitMessageGroupBatchSender( connectionManager, pinConfig.exchange, pinConfig.routingKey, - pinName + pinName, + bookName ) } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSender.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSender.kt index 99d841b6f..0f043cd9f 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSender.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSender.kt @@ -16,15 +16,14 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.group +import com.exactpro.th2.common.grpc.MessageGroup import com.exactpro.th2.common.grpc.MessageGroupBatch +import com.exactpro.th2.common.message.bookName import com.exactpro.th2.common.message.getSessionAliasAndDirection import com.exactpro.th2.common.message.toJson -import com.exactpro.th2.common.metrics.DIRECTION_LABEL -import com.exactpro.th2.common.metrics.SESSION_ALIAS_LABEL -import com.exactpro.th2.common.metrics.TH2_PIN_LABEL -import com.exactpro.th2.common.metrics.MESSAGE_TYPE_LABEL -import com.exactpro.th2.common.metrics.incrementTotalMetrics +import com.exactpro.th2.common.metrics.* import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSender +import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager import com.exactpro.th2.common.schema.message.impl.rabbitmq.group.RabbitMessageGroupBatchRouter.Companion.MESSAGE_GROUP_TYPE import io.prometheus.client.Counter @@ -34,8 +33,16 @@ class RabbitMessageGroupBatchSender( connectionManager: ConnectionManager, exchangeName: String, routingKey: String, - th2Pin: String -) : AbstractRabbitSender(connectionManager, exchangeName, routingKey, th2Pin, MESSAGE_GROUP_TYPE) { + th2Pin: String, + bookName: BookName +) : AbstractRabbitSender( + connectionManager, + exchangeName, + routingKey, + th2Pin, + MESSAGE_GROUP_TYPE, + bookName +) { override fun send(value: MessageGroupBatch) { incrementTotalMetrics( value, @@ -44,7 +51,27 @@ class RabbitMessageGroupBatchSender( MESSAGE_GROUP_PUBLISH_TOTAL, MESSAGE_GROUP_SEQUENCE_PUBLISH ) - super.send(value) + if (value.groupsList.any { group -> group.messagesList.any { message -> message.bookName.isEmpty() } }) { + val batchBuilder = MessageGroupBatch.newBuilder() + value.groupsList.forEach { messageGroup -> + val groupBuilder = MessageGroup.newBuilder() + messageGroup.messagesList.forEach { message -> + val messageBuilder = message.toBuilder() + if (message.bookName.isEmpty()) { + if (messageBuilder.hasMessage()) { + messageBuilder.messageBuilder.metadataBuilder.idBuilder.bookName = bookName + } else if (messageBuilder.hasRawMessage()) { + messageBuilder.rawMessageBuilder.metadataBuilder.idBuilder.bookName = bookName + } + } + groupBuilder.addMessages(messageBuilder) + } + batchBuilder.addGroups(groupBuilder) + } + super.send(batchBuilder.build()) + } else { + super.send(value) + } } override fun valueToBytes(value: MessageGroupBatch): ByteArray = value.toByteArray() @@ -63,7 +90,7 @@ class RabbitMessageGroupBatchSender( else -> "" } } - + companion object { private val MESSAGE_PUBLISH_TOTAL = Counter.build() .name("th2_message_publish_total") diff --git a/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java b/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java new file mode 100644 index 000000000..c6beffa8a --- /dev/null +++ b/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.builder; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.exactpro.th2.common.grpc.ConnectionID; +import com.exactpro.th2.common.grpc.Direction; +import com.exactpro.th2.common.grpc.EventID; +import com.exactpro.th2.common.grpc.MessageID; +import com.exactpro.th2.common.schema.factory.CommonFactory; + +import static com.exactpro.th2.common.message.MessageUtils.toJson; +import static com.exactpro.th2.common.schema.box.configuration.BoxConfiguration.DEFAULT_BOOK_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class MessageEventIdBuildersTest { + private static final String CONFIG_DIRECTORY = "src/test/resources/test_message_event_id_builders"; + private static final String BOOK_NAME = "book"; + private static final String DEFAULT_ALIAS = "alias"; + private static final Direction DEFAULT_DIRECTION = Direction.FIRST; + private static final int DEFAULT_SEQUENCE = 1; + private static final int DEFAULT_SUBSEQUENCE = 2; + private static final String DEFAULT_ID = "id"; + private static final String DEFAULT_SCOPE = "scope"; + + private CommonFactory commonFactory; + + @AfterEach + void tearDown() { + commonFactory.close(); + } + + @Test + public void testWithDefaultBookName() { + commonFactory = CommonFactory.createFromArguments("-c", ""); + assertIds(DEFAULT_BOOK_NAME, defaultMessageIdBuilder(), defaultEventIdBuilder()); + } + + @Test + public void testWithConfigBookName() { + commonFactory = CommonFactory.createFromArguments("-c", CONFIG_DIRECTORY); + assertIds("config_book", defaultMessageIdBuilder(), defaultEventIdBuilder()); + } + + @Test + public void testWithBookName() { + commonFactory = CommonFactory.createFromArguments("-c", CONFIG_DIRECTORY); + assertIds(BOOK_NAME, defaultMessageIdBuilder().setBookName(BOOK_NAME), defaultEventIdBuilder().setBookName(BOOK_NAME)); + } + + private void assertIds(String bookName, MessageID.Builder messageIdBuilder, EventID.Builder eventIdBuilder) { + assertEquals( + "{\n" + + " \"connectionId\": {\n" + + " \"sessionAlias\": \"" + DEFAULT_ALIAS + "\"\n" + + " },\n" + + " \"direction\": \"" + DEFAULT_DIRECTION.name() + "\",\n" + + " \"sequence\": \"" + DEFAULT_SEQUENCE + "\",\n" + + " \"subsequence\": [" + DEFAULT_SUBSEQUENCE + "],\n" + + " \"bookName\": \"" + bookName + "\"\n" + + "}", + toJson(messageIdBuilder.build(), false) + ); + assertEquals( + "{\n" + + " \"id\": \"" + DEFAULT_ID + "\",\n" + + " \"bookName\": \"" + bookName + "\",\n" + + " \"scope\": \"" + DEFAULT_SCOPE + "\"\n" + + "}", + toJson(eventIdBuilder.build(), false) + ); + } + + private MessageID.Builder defaultMessageIdBuilder() { + return commonFactory.newMessageIDBuilder() + .setConnectionId(ConnectionID.newBuilder().setSessionAlias(DEFAULT_ALIAS)) + .setDirection(DEFAULT_DIRECTION) + .setSequence(DEFAULT_SEQUENCE) + .addSubsequence(DEFAULT_SUBSEQUENCE); + } + + private EventID.Builder defaultEventIdBuilder() { + return commonFactory.newEventIDBuilder() + .setId(DEFAULT_ID) + .setScope(DEFAULT_SCOPE); + } +} diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt index 4b48e4510..56ba5e313 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt @@ -21,6 +21,7 @@ import com.exactpro.th2.common.grpc.MessageGroup import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.plusAssign +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.SubscriberMonitor import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration @@ -268,7 +269,8 @@ class TestRabbitMessageGroupBatchRouter { init(DefaultMessageRouterContext( connectionManager, mock { }, - MessageRouterConfiguration(pins) + MessageRouterConfiguration(pins), + BoxConfiguration() )) } } \ No newline at end of file diff --git a/src/test/resources/test_message_event_id_builders/box.json b/src/test/resources/test_message_event_id_builders/box.json new file mode 100644 index 000000000..20827da53 --- /dev/null +++ b/src/test/resources/test_message_event_id_builders/box.json @@ -0,0 +1,4 @@ +{ + "boxName": "config_box", + "bookName": "config_book" +} \ No newline at end of file From 6aa6b444ecf8de727f21e006a39e09e57b0ef2c4 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Tue, 30 Nov 2021 13:28:01 +0400 Subject: [PATCH 004/154] [th2-2699] added notification router (#151) * [th2-2699] added notification router * [th2-2699] AbstractCommonFactory and CommonFactory constructors with FactorySettings * [th2-2699] added setters to FactorySettings * [th2-2699] added 'globalNotification' and 'exchange' configurations to mq.json --- README.md | 8 +- .../schema/factory/AbstractCommonFactory.java | 72 +++++++++--- .../common/schema/factory/CommonFactory.java | 72 ++++++------ .../schema/message/NotificationRouter.kt | 47 ++++++++ .../connection/ConnectionManager.java | 13 ++- .../common/schema/factory/FactorySettings.kt | 106 +++++++++++++++++- .../MessageRouterConfiguration.kt | 11 +- .../NotificationEventBatchRouter.kt | 74 ++++++++++++ .../NotificationEventBatchSender.kt | 47 ++++++++ .../NotificationEventBatchSubscriber.kt | 90 +++++++++++++++ .../common/schema/TestJsonConfiguration.kt | 4 +- .../TestRabbitMessageGroupBatchRouter.kt | 3 +- 12 files changed, 486 insertions(+), 61 deletions(-) create mode 100644 src/main/java/com/exactpro/th2/common/schema/message/NotificationRouter.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSender.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt diff --git a/README.md b/README.md index 725095bed..3ebb149f4 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,10 @@ The `CommonFactory` reads a message's router configuration from the `mq.json` fi * filters - pin's message's filters * metadata - a metadata filters * message - a message's fields filters - + +* globalNotification - notification exchange in RabbitMQ + * exchange - `global-notification` by default + Filters format: * fieldName - a field's name * expectedValue - expected field's value (not used for all operations) @@ -155,6 +158,9 @@ Filters format: } ] } + }, + "globalNotification": { + "exchange": "global-notification" } } } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 28758f831..df1619d18 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -36,15 +36,14 @@ import com.exactpro.th2.common.schema.cradle.CradleConfiguration; import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration; import com.exactpro.th2.common.schema.dictionary.DictionaryType; -import com.exactpro.th2.common.schema.event.EventBatchRouter; import com.exactpro.th2.common.schema.exception.CommonFactoryException; import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration; import com.exactpro.th2.common.schema.grpc.configuration.GrpcRouterConfiguration; import com.exactpro.th2.common.schema.grpc.router.GrpcRouter; -import com.exactpro.th2.common.schema.grpc.router.impl.DefaultGrpcRouter; import com.exactpro.th2.common.schema.message.MessageRouter; import com.exactpro.th2.common.schema.message.MessageRouterContext; import com.exactpro.th2.common.schema.message.MessageRouterMonitor; +import com.exactpro.th2.common.schema.message.NotificationRouter; import com.exactpro.th2.common.schema.message.QueueAttribute; import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration; import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext; @@ -56,9 +55,6 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.MessageConverter; import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.RabbitCustomRouter; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.group.RabbitMessageGroupBatchRouter; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.parsed.RabbitParsedBatchRouter; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.raw.RabbitRawBatchRouter; import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -138,12 +134,14 @@ public abstract class AbstractCommonFactory implements AutoCloseable { private final Class> messageRouterMessageGroupBatchClass; private final Class> eventBatchRouterClass; private final Class grpcRouterClass; + private final Class> notificationEventBatchRouterClass; private final AtomicReference rabbitMqConnectionManager = new AtomicReference<>(); private final AtomicReference routerContext = new AtomicReference<>(); private final AtomicReference> messageRouterParsedBatch = new AtomicReference<>(); private final AtomicReference> messageRouterRawBatch = new AtomicReference<>(); private final AtomicReference> messageRouterMessageGroupBatch = new AtomicReference<>(); private final AtomicReference> eventBatchRouter = new AtomicReference<>(); + private final AtomicReference> notificationEventBatchRouter = new AtomicReference<>(); private final AtomicReference rootEventId = new AtomicReference<>(); private final AtomicReference grpcRouter = new AtomicReference<>(); private final AtomicReference prometheusExporter = new AtomicReference<>(); @@ -155,11 +153,27 @@ public abstract class AbstractCommonFactory implements AutoCloseable { configureLogger(); } + /** + * Create factory with non-default implementations schema classes + * @param settings {@link FactorySettings} + */ + public AbstractCommonFactory(FactorySettings settings) { + messageRouterParsedBatchClass = settings.getMessageRouterParsedBatchClass(); + messageRouterRawBatchClass = settings.getMessageRouterRawBatchClass(); + messageRouterMessageGroupBatchClass = settings.getMessageRouterMessageGroupBatchClass(); + eventBatchRouterClass = settings.getEventBatchRouterClass(); + grpcRouterClass = settings.getGrpcRouterClass(); + notificationEventBatchRouterClass = settings.getNotificationEventBatchRouterClass(); + stringSubstitutor = new StringSubstitutor(key -> defaultIfBlank(settings.getVariables().get(key), System.getenv(key))); + } + /** * Create factory with default implementation schema classes + * @deprecated Please use {@link AbstractCommonFactory#AbstractCommonFactory(FactorySettings)} */ + @Deprecated(since = "4.0.0", forRemoval = true) public AbstractCommonFactory() { - this(RabbitParsedBatchRouter.class, RabbitRawBatchRouter.class, RabbitMessageGroupBatchRouter.class, EventBatchRouter.class, DefaultGrpcRouter.class); + this(new FactorySettings()); } /** @@ -168,14 +182,21 @@ public AbstractCommonFactory() { * @param messageRouterRawBatchClass Class for {@link MessageRouter} which work with {@link RawMessageBatch} * @param eventBatchRouterClass Class for {@link MessageRouter} which work with {@link EventBatch} * @param grpcRouterClass Class for {@link GrpcRouter} + * @deprecated Please use {@link AbstractCommonFactory#AbstractCommonFactory(FactorySettings)} */ + @Deprecated(since = "4.0.0", forRemoval = true) public AbstractCommonFactory(@NotNull Class> messageRouterParsedBatchClass, @NotNull Class> messageRouterRawBatchClass, @NotNull Class> messageRouterMessageGroupBatchClass, @NotNull Class> eventBatchRouterClass, @NotNull Class grpcRouterClass) { - this(messageRouterParsedBatchClass, messageRouterRawBatchClass, messageRouterMessageGroupBatchClass, - eventBatchRouterClass, grpcRouterClass, emptyMap()); + this(new FactorySettings() + .messageRouterParsedBatchClass(messageRouterParsedBatchClass) + .messageRouterRawBatchClass(messageRouterRawBatchClass) + .messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass) + .eventBatchRouterClass(eventBatchRouterClass) + .grpcRouterClass(grpcRouterClass) + ); } /** @@ -186,19 +207,23 @@ public AbstractCommonFactory(@NotNull Class> messageRouterParsedBatchClass, @NotNull Class> messageRouterRawBatchClass, @NotNull Class> messageRouterMessageGroupBatchClass, @NotNull Class> eventBatchRouterClass, @NotNull Class grpcRouterClass, @NotNull Map environmentVariables) { - this.messageRouterParsedBatchClass = messageRouterParsedBatchClass; - this.messageRouterRawBatchClass = messageRouterRawBatchClass; - this.messageRouterMessageGroupBatchClass = messageRouterMessageGroupBatchClass; - this.eventBatchRouterClass = eventBatchRouterClass; - this.grpcRouterClass = grpcRouterClass; - this.stringSubstitutor = new StringSubstitutor(key -> defaultIfBlank(environmentVariables.get(key), System.getenv(key))); + this(new FactorySettings() + .messageRouterParsedBatchClass(messageRouterParsedBatchClass) + .messageRouterRawBatchClass(messageRouterRawBatchClass) + .messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass) + .eventBatchRouterClass(eventBatchRouterClass) + .grpcRouterClass(grpcRouterClass) + .variables(environmentVariables) + ); } public void start() { @@ -323,6 +348,25 @@ public GrpcRouter getGrpcRouter() { }); } + /** + * @return Initialized {@link NotificationRouter} which works with {@link EventBatch} + * @throws CommonFactoryException if cannot call default constructor from class + * @throws IllegalStateException if cannot read configuration + */ + public NotificationRouter getNotificationEventBatchRouter() { + return notificationEventBatchRouter.updateAndGet(router -> { + if (router == null) { + try { + router = notificationEventBatchRouterClass.getConstructor().newInstance(); + router.init(getMessageRouterContext()); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new CommonFactoryException("Can not create notification router", e); + } + } + return router; + }); + } + /** * Registers custom message router. * diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index 7084be1b4..21c12c603 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -111,6 +111,10 @@ public class CommonFactory extends AbstractCommonFactory { private static final Logger LOGGER = LoggerFactory.getLogger(CommonFactory.class.getName()); + /** + * @deprecated Please use {@link CommonFactory#CommonFactory(FactorySettings)} + */ + @Deprecated(since = "4.0.0", forRemoval = true) protected CommonFactory(Class> messageRouterParsedBatchClass, Class> messageRouterRawBatchClass, Class> messageRouterMessageGroupBatchClass, @@ -121,30 +125,28 @@ protected CommonFactory(Class> messageRout @Nullable Path oldDictionariesDir, Map environmentVariables, ConfigurationManager configurationManager) { - super(messageRouterParsedBatchClass, messageRouterRawBatchClass, messageRouterMessageGroupBatchClass, eventBatchRouterClass, grpcRouterClass, environmentVariables); - - this.custom = defaultPathIfNull(custom, CUSTOM_FILE_NAME); - this.dictionariesDir = defaultPathIfNull(dictionariesDir, DICTIONARY_DIR_NAME); - this.oldDictionariesDir = requireNonNullElse(oldDictionariesDir, CONFIG_DEFAULT_PATH); - this.configurationManager = configurationManager; - - start(); + this(new FactorySettings() + .messageRouterParsedBatchClass(messageRouterParsedBatchClass) + .messageRouterRawBatchClass(messageRouterRawBatchClass) + .messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass) + .eventBatchRouterClass(eventBatchRouterClass) + .grpcRouterClass(grpcRouterClass) + .variables(environmentVariables) + .custom(custom) + .dictionariesDir(dictionariesDir) + .oldDictionariesDir(oldDictionariesDir) + ); } public CommonFactory(FactorySettings settings) { - this(settings.getMessageRouterParsedBatchClass(), - settings.getMessageRouterRawBatchClass(), - settings.getMessageRouterMessageGroupBatchClass(), - settings.getEventBatchRouterClass(), - settings.getGrpcRouterClass(), - settings.getCustom(), - settings.getDictionariesDir(), - settings.getOldDictionariesDir(), - settings.getVariables(), - createConfigurationManager(settings)); + super(settings); + custom = defaultPathIfNull(settings.getCustom(), CUSTOM_FILE_NAME); + dictionariesDir = defaultPathIfNull(settings.getDictionariesDir(), DICTIONARY_DIR_NAME); + oldDictionariesDir = requireNonNullElse(settings.getOldDictionariesDir(), CONFIG_DEFAULT_PATH); + configurationManager = createConfigurationManager(settings); + start(); } - /** * @deprecated Please use {@link CommonFactory#CommonFactory(FactorySettings)} */ @@ -155,23 +157,21 @@ public CommonFactory(Class> messageRouterP Class> eventBatchRouterClass, Class grpcRouterClass, Path rabbitMQ, Path routerMQ, Path routerGRPC, Path cradle, Path custom, Path prometheus, Path dictionariesDir, Path boxConfiguration) { - - this(new FactorySettings(messageRouterParsedBatchClass, - messageRouterRawBatchClass, - messageRouterMessageGroupBatchClass, - eventBatchRouterClass, - grpcRouterClass, - rabbitMQ, - routerMQ, - null, - routerGRPC, - null, - cradle, - null, - prometheus, - boxConfiguration, - custom, - dictionariesDir)); + this(new FactorySettings() + .messageRouterParsedBatchClass(messageRouterParsedBatchClass) + .messageRouterRawBatchClass(messageRouterRawBatchClass) + .messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass) + .eventBatchRouterClass(eventBatchRouterClass) + .grpcRouterClass(grpcRouterClass) + .rabbitMQ(rabbitMQ) + .routerMQ(routerMQ) + .routerGRPC(routerGRPC) + .cradleConfidential(cradle) + .prometheus(prometheus) + .boxConfiguration(boxConfiguration) + .custom(custom) + .dictionariesDir(dictionariesDir) + ); } /** diff --git a/src/main/java/com/exactpro/th2/common/schema/message/NotificationRouter.kt b/src/main/java/com/exactpro/th2/common/schema/message/NotificationRouter.kt new file mode 100644 index 000000000..2d2222bd2 --- /dev/null +++ b/src/main/java/com/exactpro/th2/common/schema/message/NotificationRouter.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.message + +import com.exactpro.th2.common.schema.exception.RouterException + +/** + * Interface for send and receive RabbitMQ notification messages + * + * @param messages for send and receive + */ +interface NotificationRouter : AutoCloseable { + /** + * Initialization message router + * @param context router context + */ + fun init(context: MessageRouterContext) + + /** + * Send message to exclusive RabbitMQ queue + * + * @param message + * @throws com.exactpro.th2.common.schema.exception.RouterException if it cannot send message + */ + @Throws(RouterException::class) + fun send(message: T) + + /** + * Listen exclusive RabbitMQ queue + * + * @param callback listener + * @return SubscriberMonitor if starts listening + */ + fun subscribe(callback: MessageListener): SubscriberMonitor +} \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java index c073a711f..c434ce342 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java @@ -59,7 +59,7 @@ import java.util.function.Supplier; public class ConnectionManager implements AutoCloseable { - + public static final String EMPTY_ROUTING_KEY = ""; private static final Logger LOGGER = LoggerFactory.getLogger(ConnectionManager.class); private final Connection connection; @@ -298,6 +298,17 @@ private void basicCancel(Channel channel, String consumerTag) throws IOException channel.basicCancel(consumerTag); } + /** + * @param prefix used to build cache key but is ignored for queue declaring currently + */ + public String queueExclusiveDeclareAndBind(String prefix, String exchange) throws IOException { + Channel channel = getChannelFor(new PinId(prefix, EMPTY_ROUTING_KEY)).getChannel(); + String queue = channel.queueDeclare().getQueue(); + channel.queueBind(queue, exchange, EMPTY_ROUTING_KEY); + LOGGER.info("Declared the '{}' queue to listen to the '{}'", queue, exchange); + return queue; + } + private void shutdownSharedExecutor(int closeTimeout) { sharedExecutor.shutdown(); try { diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/factory/FactorySettings.kt b/src/main/kotlin/com/exactpro/th2/common/schema/factory/FactorySettings.kt index 8ae07513f..c9e960469 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/factory/FactorySettings.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/factory/FactorySettings.kt @@ -23,18 +23,20 @@ import com.exactpro.th2.common.schema.event.EventBatchRouter import com.exactpro.th2.common.schema.grpc.router.GrpcRouter import com.exactpro.th2.common.schema.grpc.router.impl.DefaultGrpcRouter import com.exactpro.th2.common.schema.message.MessageRouter +import com.exactpro.th2.common.schema.message.NotificationRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.group.RabbitMessageGroupBatchRouter +import com.exactpro.th2.common.schema.message.impl.rabbitmq.notification.NotificationEventBatchRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.parsed.RabbitParsedBatchRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.raw.RabbitRawBatchRouter import java.nio.file.Path - data class FactorySettings @JvmOverloads constructor( var messageRouterParsedBatchClass: Class> = RabbitParsedBatchRouter::class.java, var messageRouterRawBatchClass: Class> = RabbitRawBatchRouter::class.java, var messageRouterMessageGroupBatchClass: Class> = RabbitMessageGroupBatchRouter::class.java, var eventBatchRouterClass: Class> = EventBatchRouter::class.java, var grpcRouterClass: Class = DefaultGrpcRouter::class.java, + var notificationEventBatchRouterClass: Class> = NotificationEventBatchRouter::class.java, var rabbitMQ: Path? = null, var routerMQ: Path? = null, var connectionManagerSettings: Path? = null, @@ -46,11 +48,105 @@ data class FactorySettings @JvmOverloads constructor( var boxConfiguration: Path? = null, var custom: Path? = null, var dictionariesDir: Path? = null, - var oldDictionariesDir: Path? = null) { - private val _variables: MutableMap = HashMap() - val variables: Map = _variables + var oldDictionariesDir: Path? = null, + var variables: MutableMap = HashMap() +) { + fun messageRouterParsedBatchClass(messageRouterParsedBatchClass: Class>): FactorySettings { + this.messageRouterParsedBatchClass = messageRouterParsedBatchClass + return this + } + + fun messageRouterRawBatchClass(messageRouterRawBatchClass: Class>): FactorySettings { + this.messageRouterRawBatchClass = messageRouterRawBatchClass + return this + } + + fun messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass: Class>): FactorySettings { + this.messageRouterMessageGroupBatchClass = messageRouterMessageGroupBatchClass + return this + } + + fun eventBatchRouterClass(eventBatchRouterClass: Class>): FactorySettings { + this.eventBatchRouterClass = eventBatchRouterClass + return this + } + + fun grpcRouterClass(grpcRouterClass: Class): FactorySettings { + this.grpcRouterClass = grpcRouterClass + return this + } + + fun notificationEventBatchRouterClass(notificationEventBatchRouterClass: Class>): FactorySettings { + this.notificationEventBatchRouterClass = notificationEventBatchRouterClass + return this + } + + fun rabbitMQ(rabbitMQ: Path): FactorySettings { + this.rabbitMQ = rabbitMQ + return this + } + + fun routerMQ(routerMQ: Path): FactorySettings { + this.routerMQ = routerMQ + return this + } + + fun connectionManagerSettings(connectionManagerSettings: Path): FactorySettings { + this.connectionManagerSettings = connectionManagerSettings + return this + } + + fun grpc(grpc: Path): FactorySettings { + this.grpc = grpc + return this + } + + fun routerGRPC(routerGRPC: Path): FactorySettings { + this.routerGRPC = routerGRPC + return this + } + + fun cradleConfidential(cradleConfidential: Path): FactorySettings { + this.cradleConfidential = cradleConfidential + return this + } + + fun cradleNonConfidential(cradleNonConfidential: Path): FactorySettings { + this.cradleNonConfidential = cradleNonConfidential + return this + } + + fun prometheus(prometheus: Path): FactorySettings { + this.prometheus = prometheus + return this + } + + fun boxConfiguration(boxConfiguration: Path): FactorySettings { + this.boxConfiguration = boxConfiguration + return this + } + + fun custom(custom: Path?): FactorySettings { + this.custom = custom + return this + } + + fun dictionariesDir(dictionariesDir: Path?): FactorySettings { + this.dictionariesDir = dictionariesDir + return this + } + + fun oldDictionariesDir(oldDictionariesDir: Path?): FactorySettings { + this.oldDictionariesDir = oldDictionariesDir + return this + } + + fun variables(variables: MutableMap): FactorySettings { + this.variables = variables + return this + } fun putVariable(key: String, value: String): String? { - return _variables.put(key, value) + return variables.put(key, value) } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt index ed63c3fc2..1b5e2a39d 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt @@ -25,7 +25,10 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.commons.collections4.MultiMapUtils import org.apache.commons.collections4.MultiValuedMap -data class MessageRouterConfiguration(var queues: Map = emptyMap()) : Configuration() { +data class MessageRouterConfiguration( + var queues: Map = emptyMap(), + var globalNotification: GlobalNotificationConfiguration = GlobalNotificationConfiguration() +) : Configuration() { fun getQueueByAlias(queueAlias: String): QueueConfiguration? { return queues[queueAlias] @@ -90,4 +93,8 @@ enum class FieldFilterOperation { EMPTY, NOT_EMPTY, WILDCARD -} \ No newline at end of file +} + +data class GlobalNotificationConfiguration( + @JsonProperty var exchange: String = "global-notification" +) : Configuration() \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt new file mode 100644 index 000000000..f44f141a5 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt @@ -0,0 +1,74 @@ +/* + * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification + +import com.exactpro.th2.common.grpc.EventBatch +import com.exactpro.th2.common.schema.exception.RouterException +import com.exactpro.th2.common.schema.message.MessageListener +import com.exactpro.th2.common.schema.message.MessageRouterContext +import com.exactpro.th2.common.schema.message.NotificationRouter +import com.exactpro.th2.common.schema.message.SubscriberMonitor +import mu.KotlinLogging + +const val NOTIFICATION_QUEUE_PREFIX = "global-notification-queue" + +class NotificationEventBatchRouter : NotificationRouter { + private lateinit var queue: String + private lateinit var sender: NotificationEventBatchSender + private lateinit var subscriber: NotificationEventBatchSubscriber + + override fun init(context: MessageRouterContext) { + sender = NotificationEventBatchSender( + context.connectionManager, + context.configuration.globalNotification.exchange + ) + queue = context.connectionManager.queueExclusiveDeclareAndBind( + NOTIFICATION_QUEUE_PREFIX, + context.configuration.globalNotification.exchange + ) + subscriber = NotificationEventBatchSubscriber(context.connectionManager, queue) + } + + override fun send(message: EventBatch) { + try { + sender.send(message) + } catch (e: Exception) { + val errorMessage = "Notification cannot be send through the queue $queue" + LOGGER.error(e) { errorMessage } + throw RouterException(errorMessage) + } + } + + override fun subscribe(callback: MessageListener): SubscriberMonitor { + try { + subscriber.addListener(callback) + subscriber.start() + } catch (e: Exception) { + val errorMessage = "Listener can't be subscribed via the queue $queue" + LOGGER.error(e) { errorMessage } + throw RouterException(errorMessage) + } + return SubscriberMonitor { } + } + + override fun close() { + subscriber.close() + } + + companion object { + private val LOGGER = KotlinLogging.logger {} + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSender.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSender.kt new file mode 100644 index 000000000..0b4cd41eb --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSender.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification + +import com.exactpro.th2.common.grpc.EventBatch +import com.exactpro.th2.common.schema.message.MessageSender +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager.EMPTY_ROUTING_KEY +import java.io.IOException + +class NotificationEventBatchSender( + private val connectionManager: ConnectionManager, + private val exchange: String +) : MessageSender { + @Deprecated( + "Method is deprecated, please use constructor", + ReplaceWith("NotificationEventBatchSender()") + ) + override fun init(connectionManager: ConnectionManager, exchangeName: String, routingKey: String) { + throw UnsupportedOperationException("Method is deprecated, please use constructor") + } + + override fun send(message: EventBatch) { + try { + connectionManager.basicPublish(exchange, EMPTY_ROUTING_KEY, null, message.toByteArray()) + } catch (e: Exception) { + throw IOException( + "Can not send notification message: EventBatch: parent_event_id = ${message.parentEventId.id}", + e + ) + } + } +} diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt new file mode 100644 index 000000000..8870b854c --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt @@ -0,0 +1,90 @@ +/* + * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification + +import com.exactpro.th2.common.grpc.EventBatch +import com.exactpro.th2.common.schema.message.FilterFunction +import com.exactpro.th2.common.schema.message.MessageListener +import com.exactpro.th2.common.schema.message.MessageSubscriber +import com.exactpro.th2.common.schema.message.SubscriberMonitor +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.rabbitmq.client.Delivery +import mu.KotlinLogging +import java.util.concurrent.CopyOnWriteArrayList + +class NotificationEventBatchSubscriber( + private val connectionManager: ConnectionManager, + private val queue: String +) : MessageSubscriber { + private val listeners = CopyOnWriteArrayList>() + private lateinit var monitor: SubscriberMonitor + + @Deprecated( + "Method is deprecated, please use constructor", + ReplaceWith("NotificationEventBatchSubscriber()") + ) + override fun init(connectionManager: ConnectionManager, exchangeName: String, subscribeTargets: SubscribeTarget) { + throw UnsupportedOperationException("Method is deprecated, please use constructor") + } + + @Deprecated( + "Method is deprecated, please use constructor", + ReplaceWith("NotificationEventBatchSubscriber()") + ) + override fun init( + connectionManager: ConnectionManager, + subscribeTarget: SubscribeTarget, + filterFunc: FilterFunction + ) { + throw UnsupportedOperationException("Method is deprecated, please use constructor") + } + + override fun start() { + monitor = connectionManager.basicConsume( + queue, + { consumerTag: String, delivery: Delivery -> + for (listener in listeners) { + try { + listener.handler(consumerTag, EventBatch.parseFrom(delivery.body)) + } catch (listenerExc: Exception) { + LOGGER.warn( + "Message listener from class '{}' threw exception", + listener.javaClass, + listenerExc + ) + } + } + }, + { LOGGER.warn("Consuming cancelled for: '{}'", it) } + ) + } + + override fun addListener(messageListener: MessageListener) { + listeners.add(messageListener) + } + + override fun close() { + monitor.unsubscribe() + listeners.forEach(MessageListener::onClose) + listeners.clear() + } + + companion object { + private val LOGGER = KotlinLogging.logger {} + } +} diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index 09804e382..9bf4e24b2 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -25,6 +25,7 @@ import com.exactpro.th2.common.schema.grpc.configuration.GrpcServerConfiguration import com.exactpro.th2.common.schema.grpc.configuration.GrpcServiceConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation +import com.exactpro.th2.common.schema.message.configuration.GlobalNotificationConfiguration import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration import com.exactpro.th2.common.schema.message.configuration.MqRouterFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration @@ -196,7 +197,8 @@ class TestJsonConfiguration { ) ) ) - }) + }), + GlobalNotificationConfiguration() ) private val CRADLE_CONFIDENTIAL_CONF_JSON = loadConfJson("cradle_confidential") diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt index 56ba5e313..6bef47c81 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt @@ -26,6 +26,7 @@ import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.SubscriberMonitor import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation +import com.exactpro.th2.common.schema.message.configuration.GlobalNotificationConfiguration import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration import com.exactpro.th2.common.schema.message.configuration.MqRouterFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration @@ -269,7 +270,7 @@ class TestRabbitMessageGroupBatchRouter { init(DefaultMessageRouterContext( connectionManager, mock { }, - MessageRouterConfiguration(pins), + MessageRouterConfiguration(pins, GlobalNotificationConfiguration()), BoxConfiguration() )) } From dbc66d9cb0341cdc7aafa80accd127cff659e333 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Thu, 2 Dec 2021 17:06:37 +0400 Subject: [PATCH 005/154] Merge pull request #157 from th2-net/master_to_th2-2150-books-pages --- README.md | 16 ++ build.gradle | 8 +- .../com/exactpro/th2/common/TestUtils.kt | 158 ++++++++++++++++++ 3 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt diff --git a/README.md b/README.md index 3ebb149f4..b532b6ee7 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,19 @@ EVENTS METRICS: * th2_event_publish_total (`th2_pin`): quantity of published events * th2_event_subscribe_total (`th2_pin`): quantity of received events +###Test extensions: + +To be able to use test extensions please fill build.gradle as in example below: +```groovy +plugins { + id 'java-test-fixtures' +} + +dependencies { + testImplementation testFixtures("com.exactpro.th2:common:3.31.1") +} +``` + ## Release notes ### 4.0.0 @@ -284,6 +297,9 @@ EVENTS METRICS: + Adaptation to books/pages cradleapi 4.0.0 + Removed `cradleInstanceName` parameter from `cradle.json` +### 3.31.1 ++ Feature as test assertion methods for messages from fixtures + ### 3.31.0 + Fix printing of empty MessageGroupBatch in debug logs of MessageGroupBatch router diff --git a/build.gradle b/build.gradle index 134ea063b..83b155601 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ buildscript { } ext { - kotlin_version = "1.3.72" + kotlin_version = "1.5.30" } dependencies { @@ -15,6 +15,7 @@ buildscript { plugins { id 'java' id 'java-library' + id 'java-test-fixtures' id 'maven-publish' id "io.github.gradle-nexus.publish-plugin" version "1.0.0" id 'signing' @@ -190,7 +191,7 @@ dependencies { implementation 'com.fasterxml.uuid:java-uuid-generator:4.0.1' - implementation 'io.github.microutils:kotlin-logging:1.7.9' + implementation 'io.github.microutils:kotlin-logging:2.0.11' implementation "org.slf4j:slf4j-log4j12" implementation "org.slf4j:slf4j-api" @@ -203,6 +204,9 @@ dependencies { testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" testImplementation 'org.mockito.kotlin:mockito-kotlin:4.0.0' + + testFixturesImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit5', version: kotlin_version + testFixturesImplementation 'org.junit.jupiter:junit-jupiter:5.8.1' } jar { diff --git a/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt b/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt new file mode 100644 index 000000000..64dce0946 --- /dev/null +++ b/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt @@ -0,0 +1,158 @@ +/* + * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common + +import com.exactpro.th2.common.grpc.Message +import com.exactpro.th2.common.grpc.MessageBatch +import com.exactpro.th2.common.grpc.Value +import com.exactpro.th2.common.message.get +import com.exactpro.th2.common.message.getBigDecimal +import com.exactpro.th2.common.message.getDouble +import com.exactpro.th2.common.message.getField +import com.exactpro.th2.common.message.getInt +import com.exactpro.th2.common.message.getList +import com.exactpro.th2.common.message.getLong +import com.exactpro.th2.common.message.getMessage +import com.exactpro.th2.common.message.getString +import com.exactpro.th2.common.message.messageType +import com.google.protobuf.TextFormat +import com.google.protobuf.Timestamp +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.fail +import org.junit.platform.commons.util.StringUtils +import org.opentest4j.AssertionFailedError +import java.math.BigDecimal + +fun assertEqualBatches(expected: MessageBatch, actual: MessageBatch, lazyMessage: () -> String? = {null}) { + Assertions.assertEquals(expected.messagesCount, actual.messagesCount) {"wrong count of messages in batch: \n${TextFormat.shortDebugString(actual)}"} + expected.messagesList.forEachIndexed { i, message -> + try { + assertEqualMessages(message, actual.messagesList[i], lazyMessage) + } catch (e: AssertionFailedError) { + throw AssertionFailedError( + "Error in message from batch with index '$i'.\n${e.message}", + e.expected, + e.actual, + e.cause + ) + } + } +} + +fun assertEqualMessages(expected: Message, actual: Message, lazyMessage: () -> String? = {null}) { + val ts = Timestamp.getDefaultInstance() + val assertExpected = expected.toBuilder().apply { + metadataBuilder.timestamp = ts + }.build() + val assertActual = actual.toBuilder().apply { + metadataBuilder.timestamp = ts + }.build() + try { + Assertions.assertEquals(assertExpected, assertActual, lazyMessage) + } catch (e: AssertionFailedError) { + throw AssertionFailedError( + "Error in message with type '${actual.messageType}'.\n${e.message}", + e.expected, + e.actual, + e.cause + ) + } +} + +fun buildPrefix(message: String?): String { + return if (StringUtils.isNotBlank(message)) "$message ==> " else "" +} + +fun Message.assertContains(vararg name: String) { + name.forEach { fieldName -> + if (!this.containsFields(fieldName)) { + fail { "$messageType:$fieldName expected: not " } + } + } +} + +fun Message.assertNotContains(vararg name: String) { + name.forEach { fieldName -> + if (this.containsFields(fieldName)) { + fail { "$messageType:$fieldName expected: " } + } + } +} + +fun Message.assertField(name: String): Value { + this.assertContains(name) + return this.getField(name)!! +} + +fun Message.assertMessage(name: String): Message { + this.assertContains(name) + return this.getMessage(name)!! +} + +fun Message.assertInt(name: String, expected: Int? = null): Int { + this.assertContains(name) + val actual = this.getInt(name)!! + expected?.let { + Assertions.assertEquals(expected, actual) {"Unexpected $name field value"} + } + return actual +} + +fun Message.assertList(name: String, expected: List ? = null): List { + this.assertContains(name) + val actual = this.getList(name)!! + expected?.let { + Assertions.assertEquals(expected, actual) {"Unexpected $name field value"} + } + return actual +} + +fun Message.assertString(name: String, expected: String? = null): String { + this.assertContains(name) + val actual = this.getString(name)!! + expected?.let { + Assertions.assertEquals(expected, actual) {"Unexpected $name field value"} + } + return actual +} + +fun Message.assertDouble(name: String, expected: Double? = null): Double { + this.assertContains(name) + val actual = this.getDouble(name)!! + expected?.let { + Assertions.assertEquals(expected, actual) {"Unexpected $name field value"} + } + return actual +} + +fun Message.assertValue(name: String, expected: T? = null): T { + this.assertContains(name) + val actual = when (expected) { + is Int -> this.getInt(name) + is Double -> this.getDouble(name) + is Long -> this.getLong(name) + is BigDecimal -> this.getBigDecimal(name) + is List<*> -> this.getList(name) + is String -> this.getString(name) + null -> this[name] + else -> error("Cannot assert $name field value. Expected value type is not supported: ${expected!!::class.simpleName}") + }!! + expected?.let { + Assertions.assertEquals(expected, actual) {"Unexpected $name field value"} + } ?: Assertions.assertNull(actual) {"Unexpected $name field value"} + return actual as T +} \ No newline at end of file From 0c34c6bf5d464a07dc6bc2aded3a35fd2c18bc7c Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Thu, 2 Dec 2021 17:23:06 +0400 Subject: [PATCH 006/154] [th2-2762] added prepareStorage property to cradle.json (#155) --- README.md | 6 ++++-- .../th2/common/schema/factory/AbstractCommonFactory.java | 2 +- .../th2/common/schema/cradle/CradleConfiguration.kt | 9 ++++++--- .../exactpro/th2/common/schema/TestJsonConfiguration.kt | 3 ++- .../cradle_non_confidential.json | 3 ++- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b532b6ee7..738cf49a2 100644 --- a/README.md +++ b/README.md @@ -175,8 +175,9 @@ The `CommonFactory` reads a Cradle configuration from the cradle.json file. * password - the required setting defines the password that will be used for connecting to Cassandra. * cradleMaxEventBatchSize - this option defines the maximum event batch size in bytes with its default value set to 1048576. * cradleMaxMessageBatchSize - this option defines the maximum message batch size in bytes with its default value set to 1048576. -* timeout - this option defines connection timeout in milliseconds. If set to 0 or ommited, the default value of 5000 is used. -* pageSize - this option defines the size of the result set to fetch at a time. If set to 0 or ommited, the default value of 5000 is used. +* timeout - this option defines connection timeout in milliseconds. If set to 0 or omitted, the default value of 5000 is used. +* pageSize - this option defines the size of the result set to fetch at a time. If set to 0 or omitted, the default value of 5000 is used. +* prepareStorage - this option defines Cassandra storage can be created. If omitted, the default value false is used. ```json { @@ -296,6 +297,7 @@ dependencies { + Adaptation to books/pages cradleapi 4.0.0 + Removed `cradleInstanceName` parameter from `cradle.json` ++ Added `prepareStorage` property to `cradle.json` ### 3.31.1 + Feature as test assertion methods for messages from fixtures diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index df1619d18..94296c1fa 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -537,7 +537,7 @@ public CradleManager getCradleManager() { manager = new CassandraCradleManager( cassandraConnectionSettings, cassandraStorageSettings, - false + nonConfidentialConfiguration.getPrepareStorage() ); } catch (CradleStorageException | RuntimeException | IOException e) { throw new CommonFactoryException("Cannot create Cradle manager", e); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt index a0344935b..1ed5dca9e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt @@ -31,7 +31,8 @@ data class CradleConfiguration( var timeout: Long, var pageSize: Int, var cradleMaxEventBatchSize: Int, - var cradleMaxMessageBatchSize: Int + var cradleMaxMessageBatchSize: Int, + var prepareStorage: Boolean ) : Configuration() { constructor( cradleConfidentialConfiguration: CradleConfidentialConfiguration, @@ -46,7 +47,8 @@ data class CradleConfiguration( cradleNonConfidentialConfiguration.timeout, cradleNonConfidentialConfiguration.pageSize, cradleNonConfidentialConfiguration.cradleMaxEventBatchSize, - cradleNonConfidentialConfiguration.cradleMaxMessageBatchSize + cradleNonConfidentialConfiguration.cradleMaxMessageBatchSize, + cradleNonConfidentialConfiguration.prepareStorage ) } @@ -63,5 +65,6 @@ data class CradleNonConfidentialConfiguration( var timeout: Long = CassandraStorageSettings.DEFAULT_TIMEOUT, var pageSize: Int = 5000, var cradleMaxEventBatchSize: Int = CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE, - var cradleMaxMessageBatchSize: Int = CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE + var cradleMaxMessageBatchSize: Int = CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE, + var prepareStorage: Boolean = false ) : Configuration() \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index 9bf4e24b2..24ebd1a41 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -216,7 +216,8 @@ class TestJsonConfiguration { 888, 111, 123, - 321 + 321, + false ) private val PROMETHEUS_CONF_JSON = loadConfJson("prometheus") diff --git a/src/test/resources/test_json_configurations/cradle_non_confidential.json b/src/test/resources/test_json_configurations/cradle_non_confidential.json index df1665843..e9967c45b 100644 --- a/src/test/resources/test_json_configurations/cradle_non_confidential.json +++ b/src/test/resources/test_json_configurations/cradle_non_confidential.json @@ -2,5 +2,6 @@ "timeout": 888, "pageSize": 111, "cradleMaxEventBatchSize": 123, - "cradleMaxMessageBatchSize": 321 + "cradleMaxMessageBatchSize": 321, + "prepareStorage": false } \ No newline at end of file From 82614414c74207104b83348851be3ee64de1a7ce Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Fri, 3 Dec 2021 18:46:11 +0400 Subject: [PATCH 007/154] [th2-2743] added bookName to Event (#154) * [th2-2743] added bookName to Event * [th2-2743] removed deprecated methods from Event * [th2-2743] EventID instead of String in getRootEventId() and other places --- .../com/exactpro/th2/common/event/Event.java | 71 ++++++++----------- .../exactpro/th2/common/event/EventUtils.java | 7 +- .../schema/factory/AbstractCommonFactory.java | 29 +++++--- .../th2/common/message/MessageUtils.kt | 22 ++++-- .../schema/message/MessageRouterUtils.kt | 13 ++-- .../impl/monitor/EventMessageRouterMonitor.kt | 31 ++++---- .../builder/MessageEventIdBuildersTest.java | 7 +- .../th2/common/event/bean/BaseTest.java | 6 +- .../th2/common/event/bean/MessageTest.java | 8 ++- .../th2/common/event/bean/TableTest.java | 8 ++- .../common/event/bean/TestVerification.java | 10 ++- .../th2/common/event/bean/TreeTableTest.java | 12 ++-- .../exactpro/th2/common/event/TestEvent.kt | 70 +++++++++++++----- .../impl/TestAnyMessageFilterStrategy.kt | 10 +-- .../impl/rabbitmq/custom/TestMessageUtil.kt | 60 +++++++++++----- .../TestRabbitMessageGroupBatchRouter.kt | 13 ++-- 16 files changed, 233 insertions(+), 144 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/event/Event.java b/src/main/java/com/exactpro/th2/common/event/Event.java index dfc01e03c..be88caa1e 100644 --- a/src/main/java/com/exactpro/th2/common/event/Event.java +++ b/src/main/java/com/exactpro/th2/common/event/Event.java @@ -64,6 +64,7 @@ public class Event { protected final List attachedMessageIDS = new ArrayList<>(); protected final List body = new ArrayList<>(); protected final Instant startTimestamp; + protected String bookName; protected Instant endTimestamp; protected String type; protected String name; @@ -110,6 +111,16 @@ public static Event from(Instant startTimestamp) { .build(); } + public Event bookName(String bookName) { + if (isNotBlank(bookName)) { + if (this.bookName != null) { + throw new IllegalStateException(formatStateException("Book name", this.bookName)); + } + this.bookName = bookName; + } + return this; + } + public Event endTimestamp() { if (endTimestamp != null) { throw new IllegalStateException(formatStateException("End time", endTimestamp)); @@ -120,7 +131,7 @@ public Event endTimestamp() { /** * Sets event name if passed {@code eventName} is not blank. - * The {@link #UNKNOWN_EVENT_NAME} value will be used as default in the {@link #toProtoEvent(String)} and {@link #toProtoEvents(String)} methods if this property isn't set + * The {@link #UNKNOWN_EVENT_NAME} value will be used as default in the {@link #toProto(com.exactpro.th2.common.grpc.EventID)} and {@link #toListProto(com.exactpro.th2.common.grpc.EventID)} methods if this property isn't set * @return current event * @throws IllegalStateException if name already set */ @@ -136,7 +147,7 @@ public Event name(String eventName) { /** * Sets event description if passed {@code description} is not blank. - * This property value will be appended to the end of event name and added into event body in the {@link #toProtoEvent(String)} and {@link #toProtoEvents(String)} methods if this property isn't set + * This property value will be appended to the end of event name and added into event body in the {@link #toProto(com.exactpro.th2.common.grpc.EventID)} and {@link #toListProto(com.exactpro.th2.common.grpc.EventID)} methods if this property isn't set * @return current event * @throws IllegalStateException if description already set */ @@ -153,7 +164,7 @@ public Event description(String description) { /** * Sets event type if passed {@code eventType} is not blank. - * The {@link #UNKNOWN_EVENT_TYPE} value will be used as default in the {@link #toProtoEvent(String)} and {@link #toProtoEvents(String)} methods if this property isn't set + * The {@link #UNKNOWN_EVENT_TYPE} value will be used as default in the {@link #toProto(com.exactpro.th2.common.grpc.EventID)} and {@link #toListProto(com.exactpro.th2.common.grpc.EventID)} methods if this property isn't set * @return current event * @throws IllegalStateException if type already set */ @@ -240,27 +251,10 @@ public Event messageID(MessageID attachedMessageID) { return this; } - /** - * @deprecated prefer to use full object instead of part of them, use the {@link #toListProto(EventID)} method - */ - @Deprecated - public List toProtoEvents(@Nullable String parentID) throws JsonProcessingException { - return toListProto(toEventID(parentID)); - } - - public List toListProto(@Nullable EventID parentID) throws JsonProcessingException { return collectSubEvents(new ArrayList<>(), parentID); } - /** - * @deprecated prefer to use full object instead of part of them, use the {@link #toProto(EventID)} method - */ - @Deprecated - public com.exactpro.th2.common.grpc.Event toProtoEvent(@Nullable String parentID) throws JsonProcessingException { - return toProto(toEventID(parentID)); - } - public com.exactpro.th2.common.grpc.Event toProto(@Nullable EventID parentID) throws JsonProcessingException { if (endTimestamp == null) { endTimestamp(); @@ -271,7 +265,7 @@ public com.exactpro.th2.common.grpc.Event toProto(@Nullable EventID parentID) th .append(description); } var eventBuilder = com.exactpro.th2.common.grpc.Event.newBuilder() - .setId(toEventID(id)) + .setId(getEventId()) .setName(nameBuilder.toString()) .setType(defaultIfBlank(type, UNKNOWN_EVENT_TYPE)) .setStartTimestamp(toTimestamp(startTimestamp)) @@ -279,7 +273,7 @@ public com.exactpro.th2.common.grpc.Event toProto(@Nullable EventID parentID) th .setStatus(getAggregatedStatus().eventStatus) .setBody(ByteString.copyFrom(buildBody())); if (parentID != null) { - eventBuilder. setParentId(parentID); + eventBuilder.setParentId(parentID); } for (MessageID messageID : attachedMessageIDS) { eventBuilder.addAttachedMessageIds(messageID); @@ -332,6 +326,14 @@ public List toListBatchProto(@Nullable EventID parentID) throws Json return toBatchesProtoWithLimit(Integer.MAX_VALUE, parentID); } + public EventID getEventId() { + return toEventID(bookName, id); + } + + public String getBookName() { + return bookName; + } + public String getId() { return id; } @@ -344,18 +346,13 @@ public Instant getEndTimestamp() { return endTimestamp; } - /** - * @deprecated prefer to use full object instead of part of them, use the {@link #collectSubEvents(List, EventID)} method - */ - @Deprecated - protected List collectSubEvents(List protoEvents, @Nullable String parentID) throws JsonProcessingException { - return collectSubEvents(protoEvents, toEventID(parentID)); - } - - protected List collectSubEvents(List protoEvents, @Nullable EventID parentID) throws JsonProcessingException { - protoEvents.add(toProto(parentID)); // collect current level + protected List collectSubEvents( + List protoEvents, + @Nullable EventID parentId + ) throws JsonProcessingException { + protoEvents.add(toProto(parentId)); // collect current level for (Event subEvent : subEvents) { - subEvent.collectSubEvents(protoEvents, toEventID(id)); // collect sub level + subEvent.collectSubEvents(protoEvents, getEventId()); // collect sub level } return protoEvents; } @@ -368,14 +365,6 @@ protected String formatStateException(String fieldName, Object value) { return fieldName + " in event '" + id + "' already sed with value '" + value + '\''; } - /** - * @deprecated use {@link #getAggregatedStatus} instead - */ - @Deprecated(forRemoval = true) - protected Status getAggrigatedStatus() { - return getAggregatedStatus(); - } - @NotNull protected Status getAggregatedStatus() { if (status == Status.PASSED) { diff --git a/src/main/java/com/exactpro/th2/common/event/EventUtils.java b/src/main/java/com/exactpro/th2/common/event/EventUtils.java index 8e58fdbad..59b404fe7 100644 --- a/src/main/java/com/exactpro/th2/common/event/EventUtils.java +++ b/src/main/java/com/exactpro/th2/common/event/EventUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,12 +36,13 @@ public static Message createMessageBean(String text) { return new MessageBuilder().text(text).build(); } - @Contract("null -> null; !null -> !null") - public static @Nullable EventID toEventID(@Nullable String id) { + @Contract("_, null -> null; _, !null -> !null") + public static @Nullable EventID toEventID(String bookName, @Nullable String id) { if (id == null) { return null; } return EventID.newBuilder() + .setBookName(bookName) .setId(id) .build(); } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 94296c1fa..331c2eb7f 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -142,7 +142,7 @@ public abstract class AbstractCommonFactory implements AutoCloseable { private final AtomicReference> messageRouterMessageGroupBatch = new AtomicReference<>(); private final AtomicReference> eventBatchRouter = new AtomicReference<>(); private final AtomicReference> notificationEventBatchRouter = new AtomicReference<>(); - private final AtomicReference rootEventId = new AtomicReference<>(); + private final AtomicReference rootEventId = new AtomicReference<>(); private final AtomicReference grpcRouter = new AtomicReference<>(); private final AtomicReference prometheusExporter = new AtomicReference<>(); private final AtomicReference cradleManager = new AtomicReference<>(); @@ -627,29 +627,32 @@ public InputStream readDictionary(DictionaryType dictionaryType) { } /** - * If root event does not exist, it creates root event with its name = box name and timestamp + * If root event does not exist, it creates root event with its book name = box book name and name = box name and timestamp * @return root event id */ @Nullable - public String getRootEventId() { + public EventID getRootEventId() { return rootEventId.updateAndGet(id -> { if (id == null) { try { - String boxName = getBoxConfiguration().getBoxName(); + BoxConfiguration boxConfiguration = getBoxConfiguration(); + String boxName = boxConfiguration.getBoxName(); if (boxName == null) { return null; } - com.exactpro.th2.common.grpc.Event rootEvent = Event.start().endTimestamp() + com.exactpro.th2.common.grpc.Event rootEvent = Event.start() + .bookName(boxConfiguration.getBookName()) + .endTimestamp() .name(boxName + " " + Instant.now()) .description("Root event") .status(Event.Status.PASSED) .type("Microservice") - .toProtoEvent(null); + .toProto(null); try { getEventBatchRouter().sendAll(EventBatch.newBuilder().addEvents(rootEvent).build()); - return rootEvent.getId().getId(); + return rootEvent.getId(); } catch (IOException e) { throw new CommonFactoryException("Can not send root event", e); } @@ -682,13 +685,19 @@ protected MessageRouterContext getMessageRouterContext() { return routerContext.updateAndGet(ctx -> { if (ctx == null) { try { - MessageRouterMonitor contextMonitor; - String rootEventId = getRootEventId(); + EventID rootEventId = getRootEventId(); if (rootEventId == null) { contextMonitor = new LogMessageRouterMonitor(); } else { - contextMonitor = new BroadcastMessageRouterMonitor(new LogMessageRouterMonitor(), new EventMessageRouterMonitor(getEventBatchRouter(), rootEventId)); + contextMonitor = new BroadcastMessageRouterMonitor( + new LogMessageRouterMonitor(), + new EventMessageRouterMonitor( + getEventBatchRouter(), + rootEventId, + getBoxConfiguration().getBookName() + ) + ); } return new DefaultMessageRouterContext( diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt index c38fb6aae..7900948a6 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt @@ -53,7 +53,6 @@ import com.exactpro.th2.common.grpc.Value.KindCase.NULL_VALUE import com.exactpro.th2.common.grpc.Value.KindCase.SIMPLE_VALUE import com.exactpro.th2.common.grpc.ValueFilter import com.exactpro.th2.common.grpc.ValueOrBuilder -import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName import com.exactpro.th2.common.value.getBigDecimal import com.exactpro.th2.common.value.getBigInteger @@ -88,8 +87,9 @@ typealias FieldValueFilters = Map typealias JavaDuration = java.time.Duration fun message() : Message.Builder = Message.newBuilder() -fun message(messageType: String): Message.Builder = Message.newBuilder().setMetadata(messageType) -fun message(messageType: String, direction: Direction, sessionAlias: String): Message.Builder = Message.newBuilder().setMetadata(messageType, direction, sessionAlias) +fun message(messageType: String): Message.Builder = Message.newBuilder().setMetadata(messageType = messageType) +fun message(bookName: String, messageType: String, direction: Direction, sessionAlias: String) = + Message.newBuilder().setMetadata(bookName, messageType, direction, sessionAlias) operator fun Message.get(key: String): Value? = getField(key) fun Message.getField(fieldName: String): Value? = getFieldsOrDefault(fieldName, null) @@ -154,12 +154,12 @@ fun Message.copy(): Message.Builder = Message.newBuilder().setMetadata(metadata) fun Message.Builder.copy(): Message.Builder = Message.newBuilder().setMetadata(metadata).putAllFields(fieldsMap).setParentEventId(parentEventId) fun Message.Builder.setMetadata( + bookName: String? = null, messageType: String? = null, direction: Direction? = null, sessionAlias: String? = null, sequence: Long? = null, - timestamp: Instant? = null, - bookName: String = BoxConfiguration.DEFAULT_BOOK_NAME + timestamp: Instant? = null ): Message.Builder = setMetadata(MessageMetadata.newBuilder().also { if (messageType != null) { @@ -177,7 +177,9 @@ fun Message.Builder.setMetadata( if (sequence != null) { this.sequence = sequence } - this.bookName = bookName + if (bookName != null) { + this.bookName = bookName + } }.build() } }) @@ -295,6 +297,14 @@ fun ListValue.toListValueFilter(): ListValueFilter { } } +val Message.bookName + get(): String = metadata.id.bookName +var Message.Builder.bookName + get(): String = metadata.id.bookName + set(value) { + metadataBuilder.idBuilder.bookName = value + } + val Message.messageType get(): String = metadata.messageType var Message.Builder.messageType diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index 7afd77a54..777b71bb8 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -24,6 +24,7 @@ import com.exactpro.th2.common.event.Event.Status.PASSED import com.exactpro.th2.common.event.EventUtils import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.EventBatch +import com.exactpro.th2.common.grpc.EventID import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.message.logId import com.exactpro.th2.common.message.toJson @@ -34,18 +35,22 @@ import org.apache.commons.lang3.exception.ExceptionUtils fun MessageRouter.storeEvent( event: Event, - parentId: String? = null + parentId: EventID? = null ): Event = event.apply { - val batch = EventBatch.newBuilder().addEvents(toProtoEvent(parentId)).build() - sendAll(batch, PUBLISH.toString(), EVENT.toString()) + sendAll( + EventBatch.newBuilder().addEvents(toProto(parentId)).build(), + PUBLISH.toString(), + EVENT.toString() + ) } fun MessageRouter.storeEvent( - parentId: String, + parentId: EventID, name: String, type: String, cause: Throwable? = null ): Event = Event.start().apply { + bookName(parentId.bookName) endTimestamp() name(name) type(type) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/monitor/EventMessageRouterMonitor.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/monitor/EventMessageRouterMonitor.kt index 39991176b..5bb01b5f9 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/monitor/EventMessageRouterMonitor.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/monitor/EventMessageRouterMonitor.kt @@ -18,11 +18,16 @@ package com.exactpro.th2.common.schema.message.impl.monitor import com.exactpro.th2.common.event.Event import com.exactpro.th2.common.event.bean.Message import com.exactpro.th2.common.grpc.EventBatch +import com.exactpro.th2.common.grpc.EventID import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.MessageRouterMonitor import org.slf4j.helpers.MessageFormatter.arrayFormat -class EventMessageRouterMonitor(private val router: MessageRouter, private val parentEventID: String?) : +class EventMessageRouterMonitor( + private val router: MessageRouter, + private val parentEventID: EventID?, + private val bookName: String +) : MessageRouterMonitor { override fun onInfo(msg: String, vararg args: Any?) { @@ -37,17 +42,15 @@ class EventMessageRouterMonitor(private val router: MessageRouter, p router.send(createEventBatch("Error message in message router", arrayFormat(msg, args).message, Event.Status.FAILED)) } - private fun createEventBatch(name: String, msg: String, status: Event.Status): EventBatch = - EventBatch.newBuilder().apply { - addEvents( - Event.start() - .name(name) - .bodyData(Message().apply { data = msg; type = "message" }) - .status(status) - .type("event") - .toProtoEvent(parentEventID) - ) - }.build() - - + private fun createEventBatch(name: String, msg: String, status: Event.Status) = EventBatch.newBuilder().apply { + addEvents( + Event.start() + .bookName(bookName) + .name(name) + .bodyData(Message().apply { data = msg; type = "message" }) + .status(status) + .type("event") + .toProto(parentEventID) + ) + }.build() } \ No newline at end of file diff --git a/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java b/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java index c6beffa8a..354fa6363 100644 --- a/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java +++ b/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java @@ -26,12 +26,11 @@ import com.exactpro.th2.common.schema.factory.CommonFactory; import static com.exactpro.th2.common.message.MessageUtils.toJson; -import static com.exactpro.th2.common.schema.box.configuration.BoxConfiguration.DEFAULT_BOOK_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; public class MessageEventIdBuildersTest { private static final String CONFIG_DIRECTORY = "src/test/resources/test_message_event_id_builders"; - private static final String BOOK_NAME = "book"; + private static final String CUSTOM_BOOK = "custom_book"; private static final String DEFAULT_ALIAS = "alias"; private static final Direction DEFAULT_DIRECTION = Direction.FIRST; private static final int DEFAULT_SEQUENCE = 1; @@ -49,7 +48,7 @@ void tearDown() { @Test public void testWithDefaultBookName() { commonFactory = CommonFactory.createFromArguments("-c", ""); - assertIds(DEFAULT_BOOK_NAME, defaultMessageIdBuilder(), defaultEventIdBuilder()); + assertIds(commonFactory.getBoxConfiguration().getBookName(), defaultMessageIdBuilder(), defaultEventIdBuilder()); } @Test @@ -61,7 +60,7 @@ public void testWithConfigBookName() { @Test public void testWithBookName() { commonFactory = CommonFactory.createFromArguments("-c", CONFIG_DIRECTORY); - assertIds(BOOK_NAME, defaultMessageIdBuilder().setBookName(BOOK_NAME), defaultEventIdBuilder().setBookName(BOOK_NAME)); + assertIds(CUSTOM_BOOK, defaultMessageIdBuilder().setBookName(CUSTOM_BOOK), defaultEventIdBuilder().setBookName(CUSTOM_BOOK)); } private void assertIds(String bookName, MessageID.Builder messageIdBuilder, EventID.Builder eventIdBuilder) { diff --git a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java index 19fa758a0..9ee8b0466 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,10 @@ */ package com.exactpro.th2.common.event.bean; +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; + import org.junit.jupiter.api.Assertions; import java.io.IOException; @@ -24,7 +26,7 @@ import static com.fasterxml.jackson.module.kotlin.ExtensionsKt.jacksonObjectMapper; public class BaseTest { - + protected static final String BOOK_NAME = new BoxConfiguration().getBookName(); private static final ObjectMapper jacksonMapper = jacksonObjectMapper(); protected void assertCompareBytesAndJson(byte[] bytes, String jsonString) throws IOException { diff --git a/src/test/java/com/exactpro/th2/common/event/bean/MessageTest.java b/src/test/java/com/exactpro/th2/common/event/bean/MessageTest.java index e01cd7cf3..3c194fcfe 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/MessageTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/MessageTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,14 @@ import com.exactpro.th2.common.event.Event; import com.exactpro.th2.common.event.bean.builder.MessageBuilder; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; +import static com.exactpro.th2.common.event.EventUtils.toEventID; + public class MessageTest extends BaseTest { @Test public void testSerializationMessage() throws IOException { @@ -30,7 +33,7 @@ public void testSerializationMessage() throws IOException { .build(); com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(message).toProtoEvent("id"); + Event.start().bodyData(message).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); String expectedJson = "[\n" + " {\n" + @@ -48,5 +51,4 @@ public void testSerializationMessageNullBody() { new MessageBuilder().text(null).build(); }); } - } diff --git a/src/test/java/com/exactpro/th2/common/event/bean/TableTest.java b/src/test/java/com/exactpro/th2/common/event/bean/TableTest.java index 1d6d33d98..ff32dfb1b 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/TableTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/TableTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,13 @@ import com.exactpro.th2.common.event.Event; import com.exactpro.th2.common.event.bean.builder.TableBuilder; + import org.junit.jupiter.api.Test; import java.io.IOException; +import static com.exactpro.th2.common.event.EventUtils.toEventID; + public class TableTest extends BaseTest { @Test @@ -40,7 +43,7 @@ public void testSerializationTable() throws IOException { Table table = tableBuilder.row(row1) .row(row2).build(); com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(table).toProtoEvent("id"); + Event.start().bodyData(table).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); String expectedJson = "[\n" + " {\n" + @@ -60,5 +63,4 @@ public void testSerializationTable() throws IOException { assertCompareBytesAndJson(event.getBody().toByteArray(), expectedJson); } - } diff --git a/src/test/java/com/exactpro/th2/common/event/bean/TestVerification.java b/src/test/java/com/exactpro/th2/common/event/bean/TestVerification.java index fff144af9..8e4d38cec 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/TestVerification.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/TestVerification.java @@ -16,12 +16,18 @@ package com.exactpro.th2.common.event.bean; import com.exactpro.th2.common.event.Event; +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration; + import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.HashMap; +import static com.exactpro.th2.common.event.EventUtils.toEventID; + public class TestVerification extends BaseTest { + private static final String BOOK_NAME = new BoxConfiguration().getBookName(); + @Test public void testSerializationSimpleVerification() throws IOException { Verification verification = new Verification(); @@ -41,7 +47,7 @@ public void testSerializationSimpleVerification() throws IOException { }}); com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(verification).toProtoEvent("id"); + Event.start().bodyData(verification).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); String expectedJson = "[\n" + " {\n" + @@ -93,7 +99,7 @@ public void testSerializationRecursiveVerification() throws IOException { }}); com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(verification).toProtoEvent("id"); + Event.start().bodyData(verification).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); String expectedJson = "[\n" + " {\n" + diff --git a/src/test/java/com/exactpro/th2/common/event/bean/TreeTableTest.java b/src/test/java/com/exactpro/th2/common/event/bean/TreeTableTest.java index 7a9e998a1..0fd0c3f02 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/TreeTableTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/TreeTableTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,11 +19,13 @@ import com.exactpro.th2.common.event.bean.builder.CollectionBuilder; import com.exactpro.th2.common.event.bean.builder.RowBuilder; import com.exactpro.th2.common.event.bean.builder.TreeTableBuilder; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; +import static com.exactpro.th2.common.event.EventUtils.toEventID; public class TreeTableTest extends BaseTest { @@ -39,7 +41,7 @@ public void testSerializationRow() throws IOException { TreeTableBuilder treeTableBuilder = new TreeTableBuilder(); TreeTable treeTable = treeTableBuilder.row("FirstRow", row).build(); com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(treeTable).toProtoEvent("id"); + Event.start().bodyData(treeTable).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); String expectedJson = "[{\n" + " \"type\": \"treeTable\",\n" + @@ -81,7 +83,7 @@ public void testSerializationCollection() throws IOException { TreeTable treeTable = treeTableBuilder.row("Row B with some other name", collection).build(); com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(treeTable).toProtoEvent("id"); + Event.start().bodyData(treeTable).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); String expectedJson = "[ {\"type\": \"treeTable\",\n" + " \"rows\": {" + @@ -142,7 +144,7 @@ public void testSerializationHybrid() throws IOException { .build(); com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(treeTable).toProtoEvent("id"); + Event.start().bodyData(treeTable).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); String expectedJson = "[ {\"type\": \"treeTable\",\n" + " \"rows\": {" + @@ -193,7 +195,7 @@ public void testSerializationRecursive() throws IOException { TreeTable treeTable = treeTableBuilder.row("Row B with some other name", collection).build(); com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(treeTable).toProtoEvent("id"); + Event.start().bodyData(treeTable).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); String expectedJson = "[ {\"type\": \"treeTable\",\n" + " \"rows\": {" + diff --git a/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt b/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt index 8e25c8fed..2dadcbecd 100644 --- a/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt +++ b/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt @@ -22,6 +22,7 @@ import com.exactpro.th2.common.grpc.EventBatch import com.exactpro.th2.common.grpc.EventID import com.exactpro.th2.common.grpc.EventStatus.FAILED import com.exactpro.th2.common.grpc.EventStatus.SUCCESS +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.fasterxml.jackson.databind.ObjectMapper import com.google.protobuf.ByteString import org.junit.jupiter.api.Assertions @@ -34,20 +35,20 @@ import org.junit.jupiter.api.assertAll typealias ProtoEvent = com.exactpro.th2.common.grpc.Event class TestEvent { - - private val parentEventId: EventID = toEventID("parentEventId")!! + private val bookName = BoxConfiguration().bookName + private val parentEventId: EventID = toEventID(bookName, "parentEventId")!! private val data = EventUtils.createMessageBean("0123456789".repeat(20)) private val dataSize = MAPPER.writeValueAsBytes(listOf(data)).size private val bigData = EventUtils.createMessageBean("0123456789".repeat(30)) @Test fun `call the toProto method on a simple event`() { - Event.start().toProto(null).run { + Event.start().bookName(bookName).toProto(null).run { checkDefaultEventFields() assertFalse(hasParentId()) } - Event.start().toProto(parentEventId).run { + Event.start().bookName(bookName).toProto(parentEventId).run { checkDefaultEventFields() assertEquals(parentEventId, parentId) } @@ -55,7 +56,7 @@ class TestEvent { @Test fun `set parent to the toListProto method`() { - val event = Event.start() + val event = Event.start().bookName(bookName) val toListProtoWithParent = event.toListProto(parentEventId) val toListProtoWithoutParent = event.toListProto(null) @@ -69,7 +70,7 @@ class TestEvent { @Test fun `negative or zero max size`() { - val rootEvent = Event.start() + val rootEvent = Event.start().bookName(bookName) assertAll( { Assertions.assertThrows(IllegalArgumentException::class.java) { rootEvent.toBatchesProtoWithLimit(-1, parentEventId) } }, { Assertions.assertThrows(IllegalArgumentException::class.java) { rootEvent.toBatchesProtoWithLimit(0, parentEventId) } } @@ -79,6 +80,7 @@ class TestEvent { @Test fun `too low max size`() { val rootEvent = Event.start() + .bookName(bookName) .bodyData(data) assertAll( @@ -89,10 +91,13 @@ class TestEvent { @Test fun `every event to distinct batch`() { val rootEvent = Event.start() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) .addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) } @@ -104,10 +109,13 @@ class TestEvent { @Test fun `problem events`() { val rootEvent = Event.start() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) .addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(bigData) } @@ -119,14 +127,19 @@ class TestEvent { @Test fun `several events at the end of hierarchy`() { val rootEvent = Event.start() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(bigData) addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) } @@ -150,10 +163,13 @@ class TestEvent { @Test fun `batch structure`() { val rootEvent = Event.start() + .bookName(bookName) .bodyData(data) val subEvent1 = rootEvent.addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) val subEvent2 = rootEvent.addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) val batches = rootEvent.toBatchesProtoWithLimit(1024 * 1024, parentEventId) @@ -174,12 +190,16 @@ class TestEvent { @Test fun `event with children is after the event without children`() { val rootEvent = Event.start() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) } } @@ -193,12 +213,14 @@ class TestEvent { fun `root event to list batch proto with size limit`() { val rootName = "root" val childName = "child" - val rootEvent = Event.start().apply { - name = rootName - bodyData(data).apply { - addSubEventWithSamePeriod().apply { - name = childName - bodyData(data) + val rootEvent = Event.start().also { + it.bookName = bookName + it.name = rootName + it.bodyData(data).apply { + addSubEventWithSamePeriod().also { subEvent -> + subEvent.bookName = bookName + subEvent.name = childName + subEvent.bodyData(data) } } } @@ -215,12 +237,14 @@ class TestEvent { fun `root event to list batch proto without size limit`() { val rootName = "root" val childName = "child" - val rootEvent = Event.start().apply { - name = rootName - bodyData(data).apply { - addSubEventWithSamePeriod().apply { - name = childName - bodyData(data) + val rootEvent = Event.start().also { + it.bookName = bookName + it.name = rootName + it.bodyData(data).apply { + addSubEventWithSamePeriod().also { subEvent -> + subEvent.bookName = bookName + subEvent.name = childName + subEvent.bodyData(data) } } } @@ -233,13 +257,17 @@ class TestEvent { @Test fun `event with children is before the event without children`() { val rootEvent = Event.start() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) } addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) } @@ -251,13 +279,17 @@ class TestEvent { @Test fun `pack event tree to single batch`() { val rootEvent = Event.start() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) } addSubEventWithSamePeriod() + .bookName(bookName) .bodyData(data) } @@ -268,7 +300,7 @@ class TestEvent { @Test fun `pack single event single batch`() { - val rootEvent = Event.start() + val rootEvent = Event.start().bookName(bookName) val batch = rootEvent.toBatchProto(parentEventId) assertFalse(batch.hasParentEventId()) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt index be2cb2a9a..ebada3092 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt @@ -21,18 +21,16 @@ import com.exactpro.th2.common.grpc.Direction import com.exactpro.th2.common.grpc.RawMessage import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.toJson +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation import com.exactpro.th2.common.schema.message.configuration.MqRouterFilterConfiguration import org.apache.commons.collections4.MultiMapUtils -import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.Arguments.arguments import org.junit.jupiter.params.provider.MethodSource -import org.junit.jupiter.params.provider.ValueSource class TestAnyMessageFilterStrategy { private val strategy = AnyMessageFilterStrategy() @@ -92,8 +90,10 @@ class TestAnyMessageFilterStrategy { } companion object { + private val BOOK_NAME = BoxConfiguration().bookName + private val PARSED_MESSAGE_MATCH = AnyMessage.newBuilder().setMessage( - message("test", Direction.FIRST, "test-alias") + message(BOOK_NAME, "test", Direction.FIRST, "test-alias") ).build() private val RAW_MESSAGE_MATCH = AnyMessage.newBuilder().setRawMessage( @@ -106,7 +106,7 @@ class TestAnyMessageFilterStrategy { ).build() private val PARSED_MESSAGE_MISS_MATCH = AnyMessage.newBuilder().setMessage( - message("test1", Direction.SECOND, "test-alias1") + message(BOOK_NAME, "test1", Direction.SECOND, "test-alias1") ).build() private val RAW_MESSAGE_MISS_MATCH = AnyMessage.newBuilder().setRawMessage( diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageUtil.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageUtil.kt index 639cd343f..185ba7b69 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageUtil.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageUtil.kt @@ -18,6 +18,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.custom import com.exactpro.th2.common.grpc.Direction import com.exactpro.th2.common.grpc.Direction.SECOND import com.exactpro.th2.common.message.addField +import com.exactpro.th2.common.message.bookName import com.exactpro.th2.common.message.direction import com.exactpro.th2.common.message.fromJson import com.exactpro.th2.common.message.get @@ -34,6 +35,7 @@ import com.exactpro.th2.common.message.updateList import com.exactpro.th2.common.message.updateMessage import com.exactpro.th2.common.message.updateOrAddString import com.exactpro.th2.common.message.updateString +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.value.updateString import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse @@ -41,6 +43,7 @@ import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +private val BOOK_NAME_VALUE = BoxConfiguration().bookName private const val MESSAGE_TYPE_VALUE = "test message type" private const val SESSION_ALIAS_VALUE = "test session alias" private const val FIELD_NAME = "test field" @@ -59,7 +62,8 @@ class TestMessageUtil { assertTrue(it.hasMetadata()) assertEquals(MESSAGE_TYPE_VALUE, it.metadata.messageType) } - message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).build().also { + message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).build().also { + assertEquals(BOOK_NAME_VALUE, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -82,14 +86,30 @@ class TestMessageUtil { } } + @Test + fun `update book name`() { + val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val newBookName = builder.bookName + "Hello" + + builder.apply { + bookName = newBookName + }.also { + assertEquals(newBookName, it.bookName) + assertEquals(MESSAGE_TYPE_VALUE, it.messageType) + assertEquals(DIRECTION_VALUE, it.direction) + assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) + } + } + @Test fun `update message type`() { - val builder = message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newMessageType = builder.messageType + "Hello" builder.apply { messageType = newMessageType }.also { + assertEquals(BOOK_NAME_VALUE, it.bookName) assertEquals(newMessageType, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -98,7 +118,7 @@ class TestMessageUtil { @Test fun `update direction`() { - val builder = message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newDirection = Direction.values().asSequence() .filter{ item -> item != Direction.UNRECOGNIZED && item != builder.direction } .first() @@ -106,6 +126,7 @@ class TestMessageUtil { builder.apply { direction = newDirection }.also { + assertEquals(BOOK_NAME_VALUE, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(newDirection, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -114,12 +135,13 @@ class TestMessageUtil { @Test fun `update session alias`() { - val builder = message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newSessionAlias = builder.sessionAlias + "Hello" builder.apply { sessionAlias = newSessionAlias }.also { + assertEquals(BOOK_NAME_VALUE, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(newSessionAlias, it.sessionAlias) @@ -128,12 +150,13 @@ class TestMessageUtil { @Test fun `update sequence`() { - val builder = message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newSequence = builder.sequence++ builder.apply { sequence = newSequence }.also { + assertEquals(BOOK_NAME_VALUE, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -149,7 +172,8 @@ class TestMessageUtil { "id": { "connectionId": { "sessionAlias": "$SESSION_ALIAS_VALUE" - } + }, + "bookName": "$BOOK_NAME_VALUE" }, "messageType": "$MESSAGE_TYPE_VALUE" }, @@ -160,29 +184,31 @@ class TestMessageUtil { } } """.trimIndent()).also { - assertEquals(it.messageType, MESSAGE_TYPE_VALUE) - assertEquals(it.sessionAlias, SESSION_ALIAS_VALUE) + assertEquals(BOOK_NAME_VALUE, it.bookName) + assertEquals(MESSAGE_TYPE_VALUE, it.messageType) + assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) assertEquals(it.getString(FIELD_NAME), FIELD_VALUE) } } @Test fun `to json from json message test`() { - val json = message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + val json = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { addField(FIELD_NAME, FIELD_VALUE) }.toJson() message().fromJson(json).also { - assertEquals(it.messageType, MESSAGE_TYPE_VALUE) - assertEquals(it.direction, DIRECTION_VALUE) - assertEquals(it.sessionAlias, SESSION_ALIAS_VALUE) - assertEquals(it.getString(FIELD_NAME), FIELD_VALUE) + assertEquals(BOOK_NAME_VALUE, it.bookName) + assertEquals(MESSAGE_TYPE_VALUE, it.messageType) + assertEquals(DIRECTION_VALUE, it.direction) + assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) + assertEquals(FIELD_VALUE, it.getString(FIELD_NAME)) } } @Test fun `update field`() { - message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { addField(FIELD_NAME, FIELD_VALUE) }.updateString(FIELD_NAME) { FIELD_VALUE_2 }.also { assertEquals(it.getString(FIELD_NAME), FIELD_VALUE_2) @@ -191,7 +217,7 @@ class TestMessageUtil { @Test fun `update complex field`() { - message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { addField(FIELD_NAME, message() .addField(FIELD_NAME, FIELD_VALUE) .addField(FIELD_NAME_2, listOf(FIELD_VALUE, FIELD_VALUE_2)) @@ -207,7 +233,7 @@ class TestMessageUtil { @Test fun `update or add field test add`() { - message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { updateOrAddString(FIELD_NAME) { it ?: FIELD_VALUE } }.also { assertEquals(it.getString(FIELD_NAME), FIELD_VALUE) @@ -216,7 +242,7 @@ class TestMessageUtil { @Test fun `update or add field test update`() { - message(MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { addField(FIELD_NAME, FIELD_VALUE) updateOrAddString(FIELD_NAME) { it ?: FIELD_VALUE_2 } }.also { diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt index 6bef47c81..244dd0ee8 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt @@ -46,6 +46,7 @@ import org.mockito.kotlin.never import org.mockito.kotlin.verify class TestRabbitMessageGroupBatchRouter { + private val boxConfiguration = BoxConfiguration() private val connectionConfiguration = ConnectionManagerConfiguration() private val monitor: SubscriberMonitor = mock { } private val connectionManager: ConnectionManager = mock { @@ -103,7 +104,7 @@ class TestRabbitMessageGroupBatchRouter { router.send( MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message("test-message1", Direction.FIRST, "test-alias") } + .apply { this += message(boxConfiguration.bookName, "test-message1", Direction.FIRST, "test-alias") } ).build() ) @@ -114,7 +115,7 @@ class TestRabbitMessageGroupBatchRouter { fun `publishes to the correct pin according to attributes`() { val batch = MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message("test-message", Direction.FIRST, "test-alias") } + .apply { this += message(boxConfiguration.bookName, "test-message", Direction.FIRST, "test-alias") } ).build() router.send(batch, "test") @@ -131,7 +132,7 @@ class TestRabbitMessageGroupBatchRouter { Assertions.assertThrows(IllegalStateException::class.java) { router.send(MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message("test-message", Direction.FIRST, "test-alias") } + .apply { this += message(boxConfiguration.bookName, "test-message", Direction.FIRST, "test-alias") } ).build()) }.apply { Assertions.assertEquals( @@ -146,7 +147,7 @@ class TestRabbitMessageGroupBatchRouter { Assertions.assertThrows(IllegalStateException::class.java) { router.send(MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message("test-message", Direction.FIRST, "test-alias") } + .apply { this += message(boxConfiguration.bookName, "test-message", Direction.FIRST, "test-alias") } ).build(), "unexpected" ) @@ -162,7 +163,7 @@ class TestRabbitMessageGroupBatchRouter { fun `publishes to all correct pin according to attributes`() { val batch = MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message("test-message", Direction.FIRST, "test-alias") } + .apply { this += message(boxConfiguration.bookName, "test-message", Direction.FIRST, "test-alias") } ).build() router.sendAll(batch) @@ -271,7 +272,7 @@ class TestRabbitMessageGroupBatchRouter { connectionManager, mock { }, MessageRouterConfiguration(pins, GlobalNotificationConfiguration()), - BoxConfiguration() + boxConfiguration )) } } \ No newline at end of file From a74cdf8d8e0cc6b1fd54cfdcdc6ff5c588af19de Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Mon, 6 Dec 2021 23:04:40 +0400 Subject: [PATCH 008/154] [th2-2150] new cradle version and corrected readme (#161) --- README.md | 2 +- build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 738cf49a2..6b7aadc4e 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ dependencies { ### 4.0.0 -+ Adaptation to books/pages cradleapi 4.0.0 ++ Migration to books/pages cradle 4.0.0 + Removed `cradleInstanceName` parameter from `cradle.json` + Added `prepareStorage` property to `cradle.json` diff --git a/build.gradle b/build.gradle index 83b155601..7c1bfc08a 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-1493588338-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-1544972223-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 185fbff35c667240e140a53884a3178892212abe Mon Sep 17 00:00:00 2001 From: Andrey Drobynin Date: Mon, 6 Dec 2021 23:08:19 +0400 Subject: [PATCH 009/154] [th2-2762] added missed prepareStorage to README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f04af2999..1ce07c77e 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,8 @@ The `CommonFactory` reads a Cradle configuration from the cradle.json file. "cradleMaxEventBatchSize": 1048576, "cradleMaxMessageBatchSize": 1048576, "timeout": 5000, - "pageSize": 5000 + "pageSize": 5000, + "prepareStorage": false } ``` From 6b43a111c03098afd171e6146b6d4942cdeb8b97 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin Date: Mon, 6 Dec 2021 23:20:02 +0400 Subject: [PATCH 010/154] [th2-2150] new grpc version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7c1bfc08a..70220ab2c 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,7 @@ test { dependencies { api platform('com.exactpro.th2:bom:3.0.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'com.exactpro.th2:grpc-common:3.9.0-th2-2150-books-pages-1498965636-SNAPSHOT' + api 'com.exactpro.th2:grpc-common:3.10.0-th2-2150-books-pages-1546219925-SNAPSHOT' implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.1.12' From 35cde2b56ab63647dfc528369c5e0a5d562afad5 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Thu, 9 Dec 2021 23:17:03 +0400 Subject: [PATCH 011/154] [th2-2762] grpc with book_name_to_session_alias_to_direction_checkpoint (#168) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 70220ab2c..45a809fff 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,7 @@ test { dependencies { api platform('com.exactpro.th2:bom:3.0.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'com.exactpro.th2:grpc-common:3.10.0-th2-2150-books-pages-1546219925-SNAPSHOT' + api 'com.exactpro.th2:grpc-common:3.10.0-th2-2150-books-pages-1560044721-SNAPSHOT' implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.1.12' From 2bb408857afa3f73dc5b0a76970c598cb841f837 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Tue, 14 Dec 2021 12:33:04 +0400 Subject: [PATCH 012/154] [th2-2150] toProto by parentEventId/bookName/(bookName + scope) (#169) --- README.md | 1 + .../com/exactpro/th2/common/event/Event.java | 255 ++++++++++++++---- .../exactpro/th2/common/event/EventUtils.java | 69 ++++- .../box/configuration/BoxConfiguration.java | 8 +- .../schema/factory/AbstractCommonFactory.java | 38 +-- .../schema/message/MessageRouterUtils.kt | 16 +- .../impl/monitor/EventMessageRouterMonitor.kt | 25 +- .../th2/common/event/bean/BaseTest.java | 7 +- .../th2/common/event/bean/MessageTest.java | 8 +- .../th2/common/event/bean/TableTest.java | 8 +- .../common/event/bean/TestVerification.java | 17 +- .../th2/common/event/bean/TreeTableTest.java | 26 +- .../exactpro/th2/common/event/TestEvent.kt | 55 +--- .../impl/TestAnyMessageFilterStrategy.kt | 4 +- .../impl/rabbitmq/custom/TestMessageUtil.kt | 41 ++- .../TestRabbitMessageGroupBatchRouter.kt | 16 +- 16 files changed, 388 insertions(+), 206 deletions(-) diff --git a/README.md b/README.md index de1c642cc..e678005de 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,7 @@ dependencies { + Migration to books/pages cradle 4.0.0 + Removed `cradleInstanceName` parameter from `cradle.json` + Added `prepareStorage` property to `cradle.json` ++ `com.exactpro.th2.common.event.Event.toProto...()` by `parentEventId`/`bookName`/`(bookName + scope)` ### 3.31.4 diff --git a/src/main/java/com/exactpro/th2/common/event/Event.java b/src/main/java/com/exactpro/th2/common/event/Event.java index be88caa1e..01bc87f1c 100644 --- a/src/main/java/com/exactpro/th2/common/event/Event.java +++ b/src/main/java/com/exactpro/th2/common/event/Event.java @@ -17,12 +17,16 @@ import static com.exactpro.th2.common.event.EventUtils.createMessageBean; import static com.exactpro.th2.common.event.EventUtils.generateUUID; +import static com.exactpro.th2.common.event.EventUtils.requireNonBlankBookName; +import static com.exactpro.th2.common.event.EventUtils.requireNonBlankScope; +import static com.exactpro.th2.common.event.EventUtils.requireNonNullParentId; import static com.exactpro.th2.common.event.EventUtils.toEventID; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; import static com.google.protobuf.TextFormat.shortDebugString; import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNullElse; import static org.apache.commons.lang3.StringUtils.defaultIfBlank; +import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.time.Instant; @@ -31,7 +35,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import org.jetbrains.annotations.Contract; @@ -64,7 +67,6 @@ public class Event { protected final List attachedMessageIDS = new ArrayList<>(); protected final List body = new ArrayList<>(); protected final Instant startTimestamp; - protected String bookName; protected Instant endTimestamp; protected String type; protected String name; @@ -111,16 +113,6 @@ public static Event from(Instant startTimestamp) { .build(); } - public Event bookName(String bookName) { - if (isNotBlank(bookName)) { - if (this.bookName != null) { - throw new IllegalStateException(formatStateException("Book name", this.bookName)); - } - this.bookName = bookName; - } - return this; - } - public Event endTimestamp() { if (endTimestamp != null) { throw new IllegalStateException(formatStateException("End time", endTimestamp)); @@ -251,11 +243,84 @@ public Event messageID(MessageID attachedMessageID) { return this; } - public List toListProto(@Nullable EventID parentID) throws JsonProcessingException { - return collectSubEvents(new ArrayList<>(), parentID); + public List toListProto(@NotNull EventID parentId) throws JsonProcessingException { + return toListProto( + new ArrayList<>(), + requireNonNullParentId(parentId), + requireNonBlankBookName(parentId.getBookName()), + parentId.getScope() + ); + } + + public List toListProto(@NotNull String bookName) throws JsonProcessingException { + return toListProto( + new ArrayList<>(), + null, + requireNonBlankBookName(bookName), + null + ); + } + + public List toListProto( + @NotNull String bookName, + @NotNull String scope + ) throws JsonProcessingException { + return toListProto( + new ArrayList<>(), + null, + requireNonBlankBookName(bookName), + requireNonBlankScope(scope) + ); } - public com.exactpro.th2.common.grpc.Event toProto(@Nullable EventID parentID) throws JsonProcessingException { + private List toListProto( + List protoEvents, + @Nullable EventID parentId, + @NotNull String bookName, + @Nullable String scope + ) throws JsonProcessingException { + protoEvents.add(toProto(parentId, bookName, scope)); // collect current level + for (Event subEvent : subEvents) { + EventID eventId = isBlank(scope) + ? toEventID(bookName, id) + : toEventID(bookName, scope, id); + subEvent.toListProto(protoEvents, eventId, bookName, scope); // collect sub level + } + return protoEvents; + } + + public com.exactpro.th2.common.grpc.Event toProto(@NotNull EventID parentId) throws JsonProcessingException { + return toProto( + requireNonNullParentId(parentId), + requireNonBlankBookName(parentId.getBookName()), + parentId.getScope() + ); + } + + public com.exactpro.th2.common.grpc.Event toProto(@NotNull String bookName) throws JsonProcessingException { + return toProto( + null, + requireNonBlankBookName(bookName), + null + ); + } + + public com.exactpro.th2.common.grpc.Event toProto( + @NotNull String bookName, + @NotNull String scope + ) throws JsonProcessingException { + return toProto( + null, + requireNonBlankBookName(bookName), + requireNonBlankScope(scope) + ); + } + + private com.exactpro.th2.common.grpc.Event toProto( + @Nullable EventID parentId, + @NotNull String bookName, + @Nullable String scope + ) throws JsonProcessingException { if (endTimestamp == null) { endTimestamp(); } @@ -264,16 +329,19 @@ public com.exactpro.th2.common.grpc.Event toProto(@Nullable EventID parentID) th nameBuilder.append(" - ") .append(description); } + EventID eventId = isBlank(scope) + ? toEventID(bookName, id) + : toEventID(bookName, scope, id); var eventBuilder = com.exactpro.th2.common.grpc.Event.newBuilder() - .setId(getEventId()) + .setId(eventId) .setName(nameBuilder.toString()) .setType(defaultIfBlank(type, UNKNOWN_EVENT_TYPE)) .setStartTimestamp(toTimestamp(startTimestamp)) .setEndTimestamp(toTimestamp(endTimestamp)) .setStatus(getAggregatedStatus().eventStatus) .setBody(ByteString.copyFrom(buildBody())); - if (parentID != null) { - eventBuilder.setParentId(parentID); + if (parentId != null) { + eventBuilder.setParentId(parentId); } for (MessageID messageID : attachedMessageIDS) { eventBuilder.addAttachedMessageIds(messageID); @@ -281,57 +349,157 @@ public com.exactpro.th2.common.grpc.Event toProto(@Nullable EventID parentID) th return eventBuilder.build(); } - public EventBatch toBatchProto(@Nullable EventID parentID) throws JsonProcessingException { - List events = toListProto(parentID); + public EventBatch toBatchProto(@NotNull EventID parentId) throws JsonProcessingException { + return toBatchProto( + requireNonNullParentId(parentId), + requireNonBlankBookName(parentId.getBookName()), + parentId.getScope() + ); + } + + public EventBatch toBatchProto(@NotNull String bookName) throws JsonProcessingException { + return toBatchProto( + null, + requireNonBlankBookName(bookName), + null + ); + } + + public EventBatch toBatchProto( + @NotNull String bookName, + @NotNull String scope + ) throws JsonProcessingException { + return toBatchProto( + null, + requireNonBlankBookName(bookName), + requireNonBlankScope(scope) + ); + } + + private EventBatch toBatchProto( + @Nullable EventID parentId, + @NotNull String bookName, + @Nullable String scope + ) throws JsonProcessingException { + List events = toListProto(new ArrayList<>(), parentId, bookName, scope); EventBatch.Builder builder = EventBatch.newBuilder() .addAllEvents(events); - if (parentID != null && events.size() != 1) { - builder.setParentEventId(parentID); + if (parentId != null && events.size() != 1) { + builder.setParentEventId(parentId); } return builder.build(); } + public List toBatchesProtoWithLimit( + int maxEventBatchContentSize, + @NotNull EventID parentId + ) throws JsonProcessingException { + return toBatchesProtoWithLimit( + maxEventBatchContentSize, + requireNonNullParentId(parentId), + requireNonBlankBookName(parentId.getBookName()), + parentId.getScope() + ); + } + + public List toBatchesProtoWithLimit( + int maxEventBatchContentSize, + @NotNull String bookName + ) throws JsonProcessingException { + return toBatchesProtoWithLimit( + maxEventBatchContentSize, + null, + requireNonBlankBookName(bookName), + null + ); + } + + public List toBatchesProtoWithLimit( + int maxEventBatchContentSize, + @NotNull String bookName, + @NotNull String scope + ) throws JsonProcessingException { + return toBatchesProtoWithLimit( + maxEventBatchContentSize, + null, + requireNonBlankBookName(bookName), + requireNonBlankScope(scope) + ); + } + /** * Converts the event with all child events to a sequence of the th2 events then organizes them into batches according to event tree structure and value of max event batch content size argent. * Splitting to batch executes by principles: * * Events with children are put into distinct batches because events can't be a child of an event from another batch. * * Events without children are collected into batches according to the max size. For example, little child events can be put into one batch; big child events can be put into separate batches. * @param maxEventBatchContentSize - the maximum size of useful content in one batch which is calculated as the sum of the size of all event bodies in the batch - * @param parentID - reference to parent event for the current event tree. It may be null if the current event is root. + * @param parentId - reference to parent event for the current event tree. It may be null if the current event is root, in this case {@code bookName} is required. + * @param bookName - book name for the current event tree. It may not be null. + * @param scope - scope for the current event tree. It may be null. */ - public List toBatchesProtoWithLimit(int maxEventBatchContentSize, @Nullable EventID parentID) throws JsonProcessingException { + private List toBatchesProtoWithLimit( + int maxEventBatchContentSize, + @Nullable EventID parentId, + @NotNull String bookName, + @Nullable String scope + ) throws JsonProcessingException { if (maxEventBatchContentSize <= 0) { throw new IllegalArgumentException("'maxEventBatchContentSize' should be greater than zero, actual: " + maxEventBatchContentSize); } - List events = toListProto(parentID); + List events = toListProto(new ArrayList<>(), parentId, bookName, scope); List result = new ArrayList<>(); Map> eventGroups = events.stream().collect(Collectors.groupingBy(com.exactpro.th2.common.grpc.Event::getParentId)); - batch(maxEventBatchContentSize, result, eventGroups, parentID); + batch(maxEventBatchContentSize, result, eventGroups, parentId); return result; } + public List toListBatchProto(@NotNull EventID parentId) throws JsonProcessingException { + return toListBatchProto( + requireNonNullParentId(parentId), + requireNonBlankBookName(parentId.getBookName()), + parentId.getScope() + ); + } + + public List toListBatchProto(@NotNull String bookName) throws JsonProcessingException { + return toListBatchProto( + null, + requireNonBlankBookName(bookName), + null + ); + } + + public List toListBatchProto( + @NotNull String bookName, + @NotNull String scope + ) throws JsonProcessingException { + return toListBatchProto( + null, + requireNonBlankBookName(bookName), + requireNonBlankScope(scope) + ); + } + /** * Converts the event with all child events to a sequence of the th2 events then organizes them into batches according to event tree structure. * Splitting to batch executes by principles: * * Events with children are put into distinct batches because events can't be a child of an event from another batch. * * Events without children are collected into batches. - * @param parentID - reference to parent event for the current event tree. It may be null if the current event is root. + * @param parentId - reference to parent event for the current event tree. It may be null if the current event is root, in this case {@code bookName} is required. + * @param bookName - book name for the current event tree. It may not be null. + * @param scope - scope for the current event tree. It may be null. */ - public List toListBatchProto(@Nullable EventID parentID) throws JsonProcessingException { - return toBatchesProtoWithLimit(Integer.MAX_VALUE, parentID); - } - - public EventID getEventId() { - return toEventID(bookName, id); - } - - public String getBookName() { - return bookName; + private List toListBatchProto( + @Nullable EventID parentId, + @NotNull String bookName, + @Nullable String scope + ) throws JsonProcessingException { + return toBatchesProtoWithLimit(Integer.MAX_VALUE, parentId, bookName, scope); } public String getId() { @@ -346,17 +514,6 @@ public Instant getEndTimestamp() { return endTimestamp; } - protected List collectSubEvents( - List protoEvents, - @Nullable EventID parentId - ) throws JsonProcessingException { - protoEvents.add(toProto(parentId)); // collect current level - for (Event subEvent : subEvents) { - subEvent.collectSubEvents(protoEvents, getEventId()); // collect sub level - } - return protoEvents; - } - protected byte[] buildBody() throws JsonProcessingException { return OBJECT_MAPPER.get().writeValueAsBytes(body); } @@ -379,7 +536,7 @@ private void batch(int maxEventBatchContentSize, List result, Map events = Objects.requireNonNull(eventGroups.get(eventID), + List events = requireNonNull(eventGroups.get(eventID), eventID == DEFAULT_EVENT_ID ? "Neither of events is root event" : "Neither of events refers to " + shortDebugString(eventID)); diff --git a/src/main/java/com/exactpro/th2/common/event/EventUtils.java b/src/main/java/com/exactpro/th2/common/event/EventUtils.java index 59b404fe7..99e67128e 100644 --- a/src/main/java/com/exactpro/th2/common/event/EventUtils.java +++ b/src/main/java/com/exactpro/th2/common/event/EventUtils.java @@ -16,6 +16,7 @@ package com.exactpro.th2.common.event; import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import com.exactpro.th2.common.event.bean.Message; @@ -24,6 +25,9 @@ import com.fasterxml.uuid.Generators; import com.fasterxml.uuid.NoArgGenerator; +import static java.util.Objects.requireNonNull; +import static org.apache.commons.lang3.StringUtils.isBlank; + @SuppressWarnings("ClassNamePrefixedWithPackageName") public class EventUtils { public static final NoArgGenerator TIME_BASED_UUID_GENERATOR = Generators.timeBasedGenerator(); @@ -36,14 +40,63 @@ public static Message createMessageBean(String text) { return new MessageBuilder().text(text).build(); } - @Contract("_, null -> null; _, !null -> !null") - public static @Nullable EventID toEventID(String bookName, @Nullable String id) { - if (id == null) { - return null; + @Contract("_, null -> !null; _, !null -> !null") + public static @NotNull EventID toEventID( + @NotNull String bookName, + @Nullable String id + ) { + return internalToEventID( + requireNonBlankBookName(bookName), + null, + id + ); + } + + @Contract("_, _, null -> !null; _, _, !null -> !null") + public static @NotNull EventID toEventID( + @NotNull String bookName, + @NotNull String scope, + @Nullable String id + ) { + return internalToEventID( + requireNonBlankBookName(bookName), + requireNonBlankScope(scope), + id + ); + } + + private static @NotNull EventID internalToEventID( + @NotNull String bookName, + @Nullable String scope, + @Nullable String id + ) { + EventID.Builder builder = EventID + .newBuilder() + .setBookName(bookName); + if (scope != null) { + builder.setScope(scope); + } + if (id != null) { + builder.setId(id); + } + return builder.build(); + } + + public static EventID requireNonNullParentId(EventID parentId) { + return requireNonNull(parentId, "Parent id cannot be null"); + } + + public static String requireNonBlankBookName(String bookName) { + if (isBlank(bookName)) { + throw new IllegalArgumentException("Book name cannot be null or blank"); + } + return bookName; + } + + public static String requireNonBlankScope(String scope) { + if (isBlank(scope)) { + throw new IllegalArgumentException("Scope cannot be null or blank"); } - return EventID.newBuilder() - .setBookName(bookName) - .setId(id) - .build(); + return scope; } } diff --git a/src/main/java/com/exactpro/th2/common/schema/box/configuration/BoxConfiguration.java b/src/main/java/com/exactpro/th2/common/schema/box/configuration/BoxConfiguration.java index e46010899..d35a2db58 100644 --- a/src/main/java/com/exactpro/th2/common/schema/box/configuration/BoxConfiguration.java +++ b/src/main/java/com/exactpro/th2/common/schema/box/configuration/BoxConfiguration.java @@ -18,8 +18,11 @@ import com.exactpro.th2.common.schema.configuration.Configuration; import com.fasterxml.jackson.annotation.JsonProperty; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import static com.exactpro.th2.common.event.EventUtils.requireNonBlankBookName; + public class BoxConfiguration extends Configuration { public static final String DEFAULT_BOOK_NAME = "test_book"; @@ -38,11 +41,12 @@ public void setBoxName(@Nullable String boxName) { this.boxName = boxName; } + @NotNull public String getBookName() { return bookName; } - public void setBookName(String bookName) { - this.bookName = bookName; + public void setBookName(@NotNull String bookName) { + this.bookName = requireNonBlankBookName(bookName); } } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 331c2eb7f..bf41f7fa4 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -66,7 +66,6 @@ import org.apache.log4j.LogManager; import org.apache.log4j.PropertyConfigurator; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,7 +98,6 @@ import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_CONSISTENCY_LEVEL; import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_TIMEOUT; import static com.exactpro.th2.common.schema.util.ArchiveUtils.getGzipBase64StringDecoder; -import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.StringUtils.defaultIfBlank; @@ -630,25 +628,20 @@ public InputStream readDictionary(DictionaryType dictionaryType) { * If root event does not exist, it creates root event with its book name = box book name and name = box name and timestamp * @return root event id */ - @Nullable + @NotNull public EventID getRootEventId() { return rootEventId.updateAndGet(id -> { if (id == null) { try { BoxConfiguration boxConfiguration = getBoxConfiguration(); - String boxName = boxConfiguration.getBoxName(); - if (boxName == null) { - return null; - } - - com.exactpro.th2.common.grpc.Event rootEvent = Event.start() - .bookName(boxConfiguration.getBookName()) + com.exactpro.th2.common.grpc.Event rootEvent = Event + .start() .endTimestamp() - .name(boxName + " " + Instant.now()) + .name(boxConfiguration.getBoxName() + " " + Instant.now()) .description("Root event") .status(Event.Status.PASSED) .type("Microservice") - .toProto(null); + .toProto(boxConfiguration.getBookName()); try { getEventBatchRouter().sendAll(EventBatch.newBuilder().addEvents(rootEvent).build()); @@ -685,20 +678,13 @@ protected MessageRouterContext getMessageRouterContext() { return routerContext.updateAndGet(ctx -> { if (ctx == null) { try { - MessageRouterMonitor contextMonitor; - EventID rootEventId = getRootEventId(); - if (rootEventId == null) { - contextMonitor = new LogMessageRouterMonitor(); - } else { - contextMonitor = new BroadcastMessageRouterMonitor( - new LogMessageRouterMonitor(), - new EventMessageRouterMonitor( - getEventBatchRouter(), - rootEventId, - getBoxConfiguration().getBookName() - ) - ); - } + MessageRouterMonitor contextMonitor = new BroadcastMessageRouterMonitor( + new LogMessageRouterMonitor(), + new EventMessageRouterMonitor( + getEventBatchRouter(), + getRootEventId() + ) + ); return new DefaultMessageRouterContext( getRabbitMqConnectionManager(), diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index 777b71bb8..8480e6e8e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -35,10 +35,21 @@ import org.apache.commons.lang3.exception.ExceptionUtils fun MessageRouter.storeEvent( event: Event, - parentId: EventID? = null + parentId: EventID +) = storeEvent(event, event.toProto(parentId)) + + +fun MessageRouter.storeEvent( + event: Event, + bookName: String +) = storeEvent(event, event.toProto(bookName)) + +private fun MessageRouter.storeEvent( + event: Event, + protoEvent: com.exactpro.th2.common.grpc.Event ): Event = event.apply { sendAll( - EventBatch.newBuilder().addEvents(toProto(parentId)).build(), + EventBatch.newBuilder().addEvents(protoEvent).build(), PUBLISH.toString(), EVENT.toString() ) @@ -50,7 +61,6 @@ fun MessageRouter.storeEvent( type: String, cause: Throwable? = null ): Event = Event.start().apply { - bookName(parentId.bookName) endTimestamp() name(name) type(type) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/monitor/EventMessageRouterMonitor.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/monitor/EventMessageRouterMonitor.kt index 5bb01b5f9..0c90c3087 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/monitor/EventMessageRouterMonitor.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/monitor/EventMessageRouterMonitor.kt @@ -25,8 +25,7 @@ import org.slf4j.helpers.MessageFormatter.arrayFormat class EventMessageRouterMonitor( private val router: MessageRouter, - private val parentEventID: EventID?, - private val bookName: String + private val parentEventId: EventID ) : MessageRouterMonitor { @@ -42,15 +41,15 @@ class EventMessageRouterMonitor( router.send(createEventBatch("Error message in message router", arrayFormat(msg, args).message, Event.Status.FAILED)) } - private fun createEventBatch(name: String, msg: String, status: Event.Status) = EventBatch.newBuilder().apply { - addEvents( - Event.start() - .bookName(bookName) - .name(name) - .bodyData(Message().apply { data = msg; type = "message" }) - .status(status) - .type("event") - .toProto(parentEventID) - ) - }.build() + private fun createEventBatch(name: String, msg: String, status: Event.Status) = + EventBatch.newBuilder().also { + it.addEvents( + Event.start() + .name(name) + .bodyData(Message().apply { data = msg; type = "message" }) + .status(status) + .type("event") + .toProto(parentEventId) + ) + }.build() } \ No newline at end of file diff --git a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java index 9ee8b0466..a12903798 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java @@ -15,6 +15,7 @@ */ package com.exactpro.th2.common.event.bean; +import com.exactpro.th2.common.grpc.EventID; import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -23,10 +24,14 @@ import java.io.IOException; +import static com.exactpro.th2.common.event.EventUtils.toEventID; import static com.fasterxml.jackson.module.kotlin.ExtensionsKt.jacksonObjectMapper; public class BaseTest { - protected static final String BOOK_NAME = new BoxConfiguration().getBookName(); + public static final BoxConfiguration BOX_CONFIGURATION = new BoxConfiguration(); + public static final String BOOK_NAME = BOX_CONFIGURATION.getBookName(); + public static final EventID PARENT_EVENT_ID = toEventID(BOOK_NAME, "id"); + private static final ObjectMapper jacksonMapper = jacksonObjectMapper(); protected void assertCompareBytesAndJson(byte[] bytes, String jsonString) throws IOException { diff --git a/src/test/java/com/exactpro/th2/common/event/bean/MessageTest.java b/src/test/java/com/exactpro/th2/common/event/bean/MessageTest.java index 3c194fcfe..15992c8f5 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/MessageTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/MessageTest.java @@ -23,8 +23,6 @@ import java.io.IOException; -import static com.exactpro.th2.common.event.EventUtils.toEventID; - public class MessageTest extends BaseTest { @Test public void testSerializationMessage() throws IOException { @@ -32,8 +30,10 @@ public void testSerializationMessage() throws IOException { Message message = new MessageBuilder().text("My message for report") .build(); - com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(message).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); + com.exactpro.th2.common.grpc.Event event = Event + .start() + .bodyData(message) + .toProto(PARENT_EVENT_ID); String expectedJson = "[\n" + " {\n" + diff --git a/src/test/java/com/exactpro/th2/common/event/bean/TableTest.java b/src/test/java/com/exactpro/th2/common/event/bean/TableTest.java index ff32dfb1b..87be2e47d 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/TableTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/TableTest.java @@ -22,8 +22,6 @@ import java.io.IOException; -import static com.exactpro.th2.common.event.EventUtils.toEventID; - public class TableTest extends BaseTest { @Test @@ -42,8 +40,10 @@ public void testSerializationTable() throws IOException { TableBuilder tableBuilder = new TableBuilder<>(); Table table = tableBuilder.row(row1) .row(row2).build(); - com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(table).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); + com.exactpro.th2.common.grpc.Event event = Event + .start() + .bodyData(table) + .toProto(PARENT_EVENT_ID); String expectedJson = "[\n" + " {\n" + diff --git a/src/test/java/com/exactpro/th2/common/event/bean/TestVerification.java b/src/test/java/com/exactpro/th2/common/event/bean/TestVerification.java index 8e4d38cec..1d47494e9 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/TestVerification.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/TestVerification.java @@ -16,18 +16,13 @@ package com.exactpro.th2.common.event.bean; import com.exactpro.th2.common.event.Event; -import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration; import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.HashMap; -import static com.exactpro.th2.common.event.EventUtils.toEventID; - public class TestVerification extends BaseTest { - private static final String BOOK_NAME = new BoxConfiguration().getBookName(); - @Test public void testSerializationSimpleVerification() throws IOException { Verification verification = new Verification(); @@ -46,8 +41,10 @@ public void testSerializationSimpleVerification() throws IOException { put("Field A", verificationEntry); }}); - com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(verification).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); + com.exactpro.th2.common.grpc.Event event = Event + .start() + .bodyData(verification) + .toProto(PARENT_EVENT_ID); String expectedJson = "[\n" + " {\n" + @@ -98,8 +95,10 @@ public void testSerializationRecursiveVerification() throws IOException { put("Sub message A", verificationEntry); }}); - com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(verification).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); + com.exactpro.th2.common.grpc.Event event = Event + .start() + .bodyData(verification) + .toProto(PARENT_EVENT_ID); String expectedJson = "[\n" + " {\n" + diff --git a/src/test/java/com/exactpro/th2/common/event/bean/TreeTableTest.java b/src/test/java/com/exactpro/th2/common/event/bean/TreeTableTest.java index 0fd0c3f02..5a45e6e2c 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/TreeTableTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/TreeTableTest.java @@ -25,8 +25,6 @@ import java.io.IOException; -import static com.exactpro.th2.common.event.EventUtils.toEventID; - public class TreeTableTest extends BaseTest { @Test @@ -40,8 +38,10 @@ public void testSerializationRow() throws IOException { TreeTableBuilder treeTableBuilder = new TreeTableBuilder(); TreeTable treeTable = treeTableBuilder.row("FirstRow", row).build(); - com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(treeTable).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); + com.exactpro.th2.common.grpc.Event event = Event + .start() + .bodyData(treeTable) + .toProto(PARENT_EVENT_ID); String expectedJson = "[{\n" + " \"type\": \"treeTable\",\n" + @@ -82,8 +82,10 @@ public void testSerializationCollection() throws IOException { TreeTableBuilder treeTableBuilder = new TreeTableBuilder(); TreeTable treeTable = treeTableBuilder.row("Row B with some other name", collection).build(); - com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(treeTable).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); + com.exactpro.th2.common.grpc.Event event = Event + .start() + .bodyData(treeTable) + .toProto(PARENT_EVENT_ID); String expectedJson = "[ {\"type\": \"treeTable\",\n" + " \"rows\": {" + @@ -143,8 +145,10 @@ public void testSerializationHybrid() throws IOException { .row("FirstRow", row) .build(); - com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(treeTable).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); + com.exactpro.th2.common.grpc.Event event = Event + .start() + .bodyData(treeTable) + .toProto(PARENT_EVENT_ID); String expectedJson = "[ {\"type\": \"treeTable\",\n" + " \"rows\": {" + @@ -194,8 +198,10 @@ public void testSerializationRecursive() throws IOException { TreeTableBuilder treeTableBuilder = new TreeTableBuilder(); TreeTable treeTable = treeTableBuilder.row("Row B with some other name", collection).build(); - com.exactpro.th2.common.grpc.Event event = - Event.start().bodyData(treeTable).bookName(BOOK_NAME).toProto(toEventID(BOOK_NAME, "id")); + com.exactpro.th2.common.grpc.Event event = Event + .start() + .bodyData(treeTable) + .toProto(PARENT_EVENT_ID); String expectedJson = "[ {\"type\": \"treeTable\",\n" + " \"rows\": {" + diff --git a/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt b/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt index 2dadcbecd..04ab5131f 100644 --- a/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt +++ b/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt @@ -18,11 +18,11 @@ package com.exactpro.th2.common.event import com.exactpro.th2.common.event.Event.UNKNOWN_EVENT_NAME import com.exactpro.th2.common.event.Event.UNKNOWN_EVENT_TYPE import com.exactpro.th2.common.event.EventUtils.toEventID +import com.exactpro.th2.common.event.bean.BaseTest.BOOK_NAME import com.exactpro.th2.common.grpc.EventBatch import com.exactpro.th2.common.grpc.EventID import com.exactpro.th2.common.grpc.EventStatus.FAILED import com.exactpro.th2.common.grpc.EventStatus.SUCCESS -import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.fasterxml.jackson.databind.ObjectMapper import com.google.protobuf.ByteString import org.junit.jupiter.api.Assertions @@ -35,20 +35,19 @@ import org.junit.jupiter.api.assertAll typealias ProtoEvent = com.exactpro.th2.common.grpc.Event class TestEvent { - private val bookName = BoxConfiguration().bookName - private val parentEventId: EventID = toEventID(bookName, "parentEventId")!! + private val parentEventId: EventID = toEventID(BOOK_NAME, "parentEventId")!! private val data = EventUtils.createMessageBean("0123456789".repeat(20)) private val dataSize = MAPPER.writeValueAsBytes(listOf(data)).size private val bigData = EventUtils.createMessageBean("0123456789".repeat(30)) @Test fun `call the toProto method on a simple event`() { - Event.start().bookName(bookName).toProto(null).run { + Event.start().toProto(BOOK_NAME).run { checkDefaultEventFields() assertFalse(hasParentId()) } - Event.start().bookName(bookName).toProto(parentEventId).run { + Event.start().toProto(parentEventId).run { checkDefaultEventFields() assertEquals(parentEventId, parentId) } @@ -56,10 +55,9 @@ class TestEvent { @Test fun `set parent to the toListProto method`() { - val event = Event.start().bookName(bookName) - + val event = Event.start() val toListProtoWithParent = event.toListProto(parentEventId) - val toListProtoWithoutParent = event.toListProto(null) + val toListProtoWithoutParent = event.toListProto(BOOK_NAME) assertAll( { assertEquals(1, toListProtoWithParent.size) }, { assertEquals(1, toListProtoWithoutParent.size) }, @@ -70,7 +68,7 @@ class TestEvent { @Test fun `negative or zero max size`() { - val rootEvent = Event.start().bookName(bookName) + val rootEvent = Event.start() assertAll( { Assertions.assertThrows(IllegalArgumentException::class.java) { rootEvent.toBatchesProtoWithLimit(-1, parentEventId) } }, { Assertions.assertThrows(IllegalArgumentException::class.java) { rootEvent.toBatchesProtoWithLimit(0, parentEventId) } } @@ -80,7 +78,6 @@ class TestEvent { @Test fun `too low max size`() { val rootEvent = Event.start() - .bookName(bookName) .bodyData(data) assertAll( @@ -91,13 +88,10 @@ class TestEvent { @Test fun `every event to distinct batch`() { val rootEvent = Event.start() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) .addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) } @@ -109,13 +103,10 @@ class TestEvent { @Test fun `problem events`() { val rootEvent = Event.start() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) .addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(bigData) } @@ -127,19 +118,14 @@ class TestEvent { @Test fun `several events at the end of hierarchy`() { val rootEvent = Event.start() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(bigData) addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) } @@ -163,13 +149,10 @@ class TestEvent { @Test fun `batch structure`() { val rootEvent = Event.start() - .bookName(bookName) .bodyData(data) val subEvent1 = rootEvent.addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) val subEvent2 = rootEvent.addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) val batches = rootEvent.toBatchesProtoWithLimit(1024 * 1024, parentEventId) @@ -190,16 +173,12 @@ class TestEvent { @Test fun `event with children is after the event without children`() { val rootEvent = Event.start() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) } } @@ -214,18 +193,16 @@ class TestEvent { val rootName = "root" val childName = "child" val rootEvent = Event.start().also { - it.bookName = bookName it.name = rootName it.bodyData(data).apply { addSubEventWithSamePeriod().also { subEvent -> - subEvent.bookName = bookName subEvent.name = childName subEvent.bodyData(data) } } } - val batches = rootEvent.toBatchesProtoWithLimit(dataSize, null) + val batches = rootEvent.toBatchesProtoWithLimit(dataSize, BOOK_NAME) assertEquals(2, batches.size) checkEventStatus(batches, 2, 0) @@ -238,18 +215,16 @@ class TestEvent { val rootName = "root" val childName = "child" val rootEvent = Event.start().also { - it.bookName = bookName it.name = rootName it.bodyData(data).apply { addSubEventWithSamePeriod().also { subEvent -> - subEvent.bookName = bookName subEvent.name = childName subEvent.bodyData(data) } } } - val batch = rootEvent.toBatchProto(null) + val batch = rootEvent.toBatchProto(BOOK_NAME) checkEventStatus(listOf(batch), 2, 0) batch.checkEventBatch(false, listOf(rootName, childName)) } @@ -257,17 +232,13 @@ class TestEvent { @Test fun `event with children is before the event without children`() { val rootEvent = Event.start() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) } addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) } @@ -279,17 +250,13 @@ class TestEvent { @Test fun `pack event tree to single batch`() { val rootEvent = Event.start() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data).apply { addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) } addSubEventWithSamePeriod() - .bookName(bookName) .bodyData(data) } @@ -300,9 +267,7 @@ class TestEvent { @Test fun `pack single event single batch`() { - val rootEvent = Event.start().bookName(bookName) - - val batch = rootEvent.toBatchProto(parentEventId) + val batch = Event.start().toBatchProto(parentEventId) assertFalse(batch.hasParentEventId()) checkEventStatus(listOf(batch), 1, 0) } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt index ebada3092..c1e7ada7e 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt @@ -16,12 +16,12 @@ package com.exactpro.th2.common.schema.filter.strategy.impl +import com.exactpro.th2.common.event.bean.BaseTest.BOOK_NAME import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.Direction import com.exactpro.th2.common.grpc.RawMessage import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.toJson -import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation import com.exactpro.th2.common.schema.message.configuration.MqRouterFilterConfiguration @@ -90,8 +90,6 @@ class TestAnyMessageFilterStrategy { } companion object { - private val BOOK_NAME = BoxConfiguration().bookName - private val PARSED_MESSAGE_MATCH = AnyMessage.newBuilder().setMessage( message(BOOK_NAME, "test", Direction.FIRST, "test-alias") ).build() diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageUtil.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageUtil.kt index 185ba7b69..1a4b2fb2d 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageUtil.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageUtil.kt @@ -15,6 +15,7 @@ */ package com.exactpro.th2.common.schema.message.impl.rabbitmq.custom +import com.exactpro.th2.common.event.bean.BaseTest.BOOK_NAME import com.exactpro.th2.common.grpc.Direction import com.exactpro.th2.common.grpc.Direction.SECOND import com.exactpro.th2.common.message.addField @@ -35,7 +36,6 @@ import com.exactpro.th2.common.message.updateList import com.exactpro.th2.common.message.updateMessage import com.exactpro.th2.common.message.updateOrAddString import com.exactpro.th2.common.message.updateString -import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.value.updateString import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse @@ -43,7 +43,6 @@ import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -private val BOOK_NAME_VALUE = BoxConfiguration().bookName private const val MESSAGE_TYPE_VALUE = "test message type" private const val SESSION_ALIAS_VALUE = "test session alias" private const val FIELD_NAME = "test field" @@ -62,8 +61,8 @@ class TestMessageUtil { assertTrue(it.hasMetadata()) assertEquals(MESSAGE_TYPE_VALUE, it.metadata.messageType) } - message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).build().also { - assertEquals(BOOK_NAME_VALUE, it.bookName) + message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).build().also { + assertEquals(BOOK_NAME, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -88,7 +87,7 @@ class TestMessageUtil { @Test fun `update book name`() { - val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newBookName = builder.bookName + "Hello" builder.apply { @@ -103,13 +102,13 @@ class TestMessageUtil { @Test fun `update message type`() { - val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newMessageType = builder.messageType + "Hello" builder.apply { messageType = newMessageType }.also { - assertEquals(BOOK_NAME_VALUE, it.bookName) + assertEquals(BOOK_NAME, it.bookName) assertEquals(newMessageType, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -118,7 +117,7 @@ class TestMessageUtil { @Test fun `update direction`() { - val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newDirection = Direction.values().asSequence() .filter{ item -> item != Direction.UNRECOGNIZED && item != builder.direction } .first() @@ -126,7 +125,7 @@ class TestMessageUtil { builder.apply { direction = newDirection }.also { - assertEquals(BOOK_NAME_VALUE, it.bookName) + assertEquals(BOOK_NAME, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(newDirection, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -135,13 +134,13 @@ class TestMessageUtil { @Test fun `update session alias`() { - val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newSessionAlias = builder.sessionAlias + "Hello" builder.apply { sessionAlias = newSessionAlias }.also { - assertEquals(BOOK_NAME_VALUE, it.bookName) + assertEquals(BOOK_NAME, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(newSessionAlias, it.sessionAlias) @@ -150,13 +149,13 @@ class TestMessageUtil { @Test fun `update sequence`() { - val builder = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) + val builder = message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE) val newSequence = builder.sequence++ builder.apply { sequence = newSequence }.also { - assertEquals(BOOK_NAME_VALUE, it.bookName) + assertEquals(BOOK_NAME, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -173,7 +172,7 @@ class TestMessageUtil { "connectionId": { "sessionAlias": "$SESSION_ALIAS_VALUE" }, - "bookName": "$BOOK_NAME_VALUE" + "bookName": "$BOOK_NAME" }, "messageType": "$MESSAGE_TYPE_VALUE" }, @@ -184,7 +183,7 @@ class TestMessageUtil { } } """.trimIndent()).also { - assertEquals(BOOK_NAME_VALUE, it.bookName) + assertEquals(BOOK_NAME, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) assertEquals(it.getString(FIELD_NAME), FIELD_VALUE) @@ -193,12 +192,12 @@ class TestMessageUtil { @Test fun `to json from json message test`() { - val json = message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + val json = message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { addField(FIELD_NAME, FIELD_VALUE) }.toJson() message().fromJson(json).also { - assertEquals(BOOK_NAME_VALUE, it.bookName) + assertEquals(BOOK_NAME, it.bookName) assertEquals(MESSAGE_TYPE_VALUE, it.messageType) assertEquals(DIRECTION_VALUE, it.direction) assertEquals(SESSION_ALIAS_VALUE, it.sessionAlias) @@ -208,7 +207,7 @@ class TestMessageUtil { @Test fun `update field`() { - message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { addField(FIELD_NAME, FIELD_VALUE) }.updateString(FIELD_NAME) { FIELD_VALUE_2 }.also { assertEquals(it.getString(FIELD_NAME), FIELD_VALUE_2) @@ -217,7 +216,7 @@ class TestMessageUtil { @Test fun `update complex field`() { - message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { addField(FIELD_NAME, message() .addField(FIELD_NAME, FIELD_VALUE) .addField(FIELD_NAME_2, listOf(FIELD_VALUE, FIELD_VALUE_2)) @@ -233,7 +232,7 @@ class TestMessageUtil { @Test fun `update or add field test add`() { - message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { updateOrAddString(FIELD_NAME) { it ?: FIELD_VALUE } }.also { assertEquals(it.getString(FIELD_NAME), FIELD_VALUE) @@ -242,7 +241,7 @@ class TestMessageUtil { @Test fun `update or add field test update`() { - message(BOOK_NAME_VALUE, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { + message(BOOK_NAME, MESSAGE_TYPE_VALUE, DIRECTION_VALUE, SESSION_ALIAS_VALUE).apply { addField(FIELD_NAME, FIELD_VALUE) updateOrAddString(FIELD_NAME) { it ?: FIELD_VALUE_2 } }.also { diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt index 244dd0ee8..6010dd274 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt @@ -16,12 +16,13 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.group +import com.exactpro.th2.common.event.bean.BaseTest.BOOK_NAME +import com.exactpro.th2.common.event.bean.BaseTest.BOX_CONFIGURATION import com.exactpro.th2.common.grpc.Direction import com.exactpro.th2.common.grpc.MessageGroup import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.plusAssign -import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.SubscriberMonitor import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration @@ -46,7 +47,6 @@ import org.mockito.kotlin.never import org.mockito.kotlin.verify class TestRabbitMessageGroupBatchRouter { - private val boxConfiguration = BoxConfiguration() private val connectionConfiguration = ConnectionManagerConfiguration() private val monitor: SubscriberMonitor = mock { } private val connectionManager: ConnectionManager = mock { @@ -104,7 +104,7 @@ class TestRabbitMessageGroupBatchRouter { router.send( MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message(boxConfiguration.bookName, "test-message1", Direction.FIRST, "test-alias") } + .apply { this += message(BOOK_NAME, "test-message1", Direction.FIRST, "test-alias") } ).build() ) @@ -115,7 +115,7 @@ class TestRabbitMessageGroupBatchRouter { fun `publishes to the correct pin according to attributes`() { val batch = MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message(boxConfiguration.bookName, "test-message", Direction.FIRST, "test-alias") } + .apply { this += message(BOOK_NAME, "test-message", Direction.FIRST, "test-alias") } ).build() router.send(batch, "test") @@ -132,7 +132,7 @@ class TestRabbitMessageGroupBatchRouter { Assertions.assertThrows(IllegalStateException::class.java) { router.send(MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message(boxConfiguration.bookName, "test-message", Direction.FIRST, "test-alias") } + .apply { this += message(BOOK_NAME, "test-message", Direction.FIRST, "test-alias") } ).build()) }.apply { Assertions.assertEquals( @@ -147,7 +147,7 @@ class TestRabbitMessageGroupBatchRouter { Assertions.assertThrows(IllegalStateException::class.java) { router.send(MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message(boxConfiguration.bookName, "test-message", Direction.FIRST, "test-alias") } + .apply { this += message(BOOK_NAME, "test-message", Direction.FIRST, "test-alias") } ).build(), "unexpected" ) @@ -163,7 +163,7 @@ class TestRabbitMessageGroupBatchRouter { fun `publishes to all correct pin according to attributes`() { val batch = MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() - .apply { this += message(boxConfiguration.bookName, "test-message", Direction.FIRST, "test-alias") } + .apply { this += message(BOOK_NAME, "test-message", Direction.FIRST, "test-alias") } ).build() router.sendAll(batch) @@ -272,7 +272,7 @@ class TestRabbitMessageGroupBatchRouter { connectionManager, mock { }, MessageRouterConfiguration(pins, GlobalNotificationConfiguration()), - boxConfiguration + BOX_CONFIGURATION )) } } \ No newline at end of file From 422d99b1e760c99ca2bee24e9a71bc2999ad9cd4 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin Date: Mon, 20 Dec 2021 18:28:05 +0400 Subject: [PATCH 013/154] [th2-2876] added default scope to EventID --- .../com/exactpro/th2/common/event/EventUtils.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/event/EventUtils.java b/src/main/java/com/exactpro/th2/common/event/EventUtils.java index 99e67128e..d263f77c6 100644 --- a/src/main/java/com/exactpro/th2/common/event/EventUtils.java +++ b/src/main/java/com/exactpro/th2/common/event/EventUtils.java @@ -31,6 +31,7 @@ @SuppressWarnings("ClassNamePrefixedWithPackageName") public class EventUtils { public static final NoArgGenerator TIME_BASED_UUID_GENERATOR = Generators.timeBasedGenerator(); + public static final String DEFAULT_SCOPE = "th2-scope"; public static String generateUUID() { return TIME_BASED_UUID_GENERATOR.generate().toString(); @@ -47,7 +48,7 @@ public static Message createMessageBean(String text) { ) { return internalToEventID( requireNonBlankBookName(bookName), - null, + DEFAULT_SCOPE, id ); } @@ -67,15 +68,13 @@ public static Message createMessageBean(String text) { private static @NotNull EventID internalToEventID( @NotNull String bookName, - @Nullable String scope, + @NotNull String scope, @Nullable String id ) { EventID.Builder builder = EventID .newBuilder() - .setBookName(bookName); - if (scope != null) { - builder.setScope(scope); - } + .setBookName(bookName) + .setScope(scope); if (id != null) { builder.setId(id); } From c99ecb80e6b1a0324d630ba6ba9b18fc5bd5bd1d Mon Sep 17 00:00:00 2001 From: Andrey Drobynin Date: Mon, 20 Dec 2021 19:25:01 +0400 Subject: [PATCH 014/154] [th2-2876] moved startTimestamp to EventID --- build.gradle | 2 +- .../com/exactpro/th2/common/event/Event.java | 35 +++++++------------ .../exactpro/th2/common/event/EventUtils.java | 26 ++++++++++++-- .../th2/common/event/bean/BaseTest.java | 3 +- .../exactpro/th2/common/event/TestEvent.kt | 4 +-- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/build.gradle b/build.gradle index 45a809fff..5c1d555bb 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,7 @@ test { dependencies { api platform('com.exactpro.th2:bom:3.0.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'com.exactpro.th2:grpc-common:3.10.0-th2-2150-books-pages-1560044721-SNAPSHOT' + api 'com.exactpro.th2:grpc-common:4.0.0-th2-2150-books-pages-1606698156-SNAPSHOT' implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.1.12' diff --git a/src/main/java/com/exactpro/th2/common/event/Event.java b/src/main/java/com/exactpro/th2/common/event/Event.java index 01bc87f1c..73617d0a0 100644 --- a/src/main/java/com/exactpro/th2/common/event/Event.java +++ b/src/main/java/com/exactpro/th2/common/event/Event.java @@ -20,7 +20,9 @@ import static com.exactpro.th2.common.event.EventUtils.requireNonBlankBookName; import static com.exactpro.th2.common.event.EventUtils.requireNonBlankScope; import static com.exactpro.th2.common.event.EventUtils.requireNonNullParentId; +import static com.exactpro.th2.common.event.EventUtils.requireNonNullTimestamp; import static com.exactpro.th2.common.event.EventUtils.toEventID; +import static com.exactpro.th2.common.event.EventUtils.toTimestamp; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; import static com.google.protobuf.TextFormat.shortDebugString; import static java.util.Objects.requireNonNull; @@ -37,7 +39,6 @@ import java.util.Map; import java.util.stream.Collectors; -import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -51,7 +52,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; public class Event { private static final Logger LOGGER = LoggerFactory.getLogger(Event.class); @@ -73,12 +73,15 @@ public class Event { protected String description; protected Status status = Status.PASSED; - protected Event(Instant startTimestamp, @Nullable Instant endTimestamp) { - this.startTimestamp = startTimestamp; + protected Event( + @NotNull Instant startTimestamp, + @Nullable Instant endTimestamp + ) { + this.startTimestamp = requireNonNullTimestamp(startTimestamp); this.endTimestamp = endTimestamp; } - protected Event(Instant startTimestamp) { + protected Event(@NotNull Instant startTimestamp) { this(startTimestamp, null); } @@ -98,21 +101,10 @@ public static Event start() { * Creates event with passed time as start * @return new event */ - public static Event from(Instant startTimestamp) { + public static Event from(@NotNull Instant startTimestamp) { return new Event(startTimestamp); } - @Contract("null -> null") - private static @Nullable Timestamp toTimestamp(@Nullable Instant instant) { - if (instant == null) { - return null; - } - return Timestamp.newBuilder() - .setSeconds(instant.getEpochSecond()) - .setNanos(instant.getNano()) - .build(); - } - public Event endTimestamp() { if (endTimestamp != null) { throw new IllegalStateException(formatStateException("End time", endTimestamp)); @@ -282,8 +274,8 @@ private List toListProto( protoEvents.add(toProto(parentId, bookName, scope)); // collect current level for (Event subEvent : subEvents) { EventID eventId = isBlank(scope) - ? toEventID(bookName, id) - : toEventID(bookName, scope, id); + ? toEventID(startTimestamp, bookName, id) + : toEventID(startTimestamp, bookName, scope, id); subEvent.toListProto(protoEvents, eventId, bookName, scope); // collect sub level } return protoEvents; @@ -330,13 +322,12 @@ private com.exactpro.th2.common.grpc.Event toProto( .append(description); } EventID eventId = isBlank(scope) - ? toEventID(bookName, id) - : toEventID(bookName, scope, id); + ? toEventID(startTimestamp, bookName, id) + : toEventID(startTimestamp, bookName, scope, id); var eventBuilder = com.exactpro.th2.common.grpc.Event.newBuilder() .setId(eventId) .setName(nameBuilder.toString()) .setType(defaultIfBlank(type, UNKNOWN_EVENT_TYPE)) - .setStartTimestamp(toTimestamp(startTimestamp)) .setEndTimestamp(toTimestamp(endTimestamp)) .setStatus(getAggregatedStatus().eventStatus) .setBody(ByteString.copyFrom(buildBody())); diff --git a/src/main/java/com/exactpro/th2/common/event/EventUtils.java b/src/main/java/com/exactpro/th2/common/event/EventUtils.java index d263f77c6..be452af47 100644 --- a/src/main/java/com/exactpro/th2/common/event/EventUtils.java +++ b/src/main/java/com/exactpro/th2/common/event/EventUtils.java @@ -15,6 +15,8 @@ */ package com.exactpro.th2.common.event; +import java.time.Instant; + import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -24,6 +26,7 @@ import com.exactpro.th2.common.grpc.EventID; import com.fasterxml.uuid.Generators; import com.fasterxml.uuid.NoArgGenerator; +import com.google.protobuf.Timestamp; import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -41,25 +44,29 @@ public static Message createMessageBean(String text) { return new MessageBuilder().text(text).build(); } - @Contract("_, null -> !null; _, !null -> !null") + @Contract("_, _, null -> !null; _, _, !null -> !null") public static @NotNull EventID toEventID( + @NotNull Instant startTimestamp, @NotNull String bookName, @Nullable String id ) { return internalToEventID( + startTimestamp, requireNonBlankBookName(bookName), DEFAULT_SCOPE, id ); } - @Contract("_, _, null -> !null; _, _, !null -> !null") + @Contract("_, _, _, null -> !null; _, _, _, !null -> !null") public static @NotNull EventID toEventID( + @NotNull Instant startTimestamp, @NotNull String bookName, @NotNull String scope, @Nullable String id ) { return internalToEventID( + startTimestamp, requireNonBlankBookName(bookName), requireNonBlankScope(scope), id @@ -67,12 +74,14 @@ public static Message createMessageBean(String text) { } private static @NotNull EventID internalToEventID( + @NotNull Instant startTimestamp, @NotNull String bookName, @NotNull String scope, @Nullable String id ) { EventID.Builder builder = EventID .newBuilder() + .setStartTimestamp(toTimestamp(startTimestamp)) .setBookName(bookName) .setScope(scope); if (id != null) { @@ -81,6 +90,19 @@ public static Message createMessageBean(String text) { return builder.build(); } + public static @NotNull Timestamp toTimestamp(@NotNull Instant timestamp) { + requireNonNullTimestamp(timestamp); + return Timestamp + .newBuilder() + .setSeconds(timestamp.getEpochSecond()) + .setNanos(timestamp.getNano()) + .build(); + } + + public static Instant requireNonNullTimestamp(Instant timestamp) { + return requireNonNull(timestamp, "Timestamp cannot be null"); + } + public static EventID requireNonNullParentId(EventID parentId) { return requireNonNull(parentId, "Parent id cannot be null"); } diff --git a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java index a12903798..ef7cbfcf8 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.Assertions; import java.io.IOException; +import java.time.Instant; import static com.exactpro.th2.common.event.EventUtils.toEventID; import static com.fasterxml.jackson.module.kotlin.ExtensionsKt.jacksonObjectMapper; @@ -30,7 +31,7 @@ public class BaseTest { public static final BoxConfiguration BOX_CONFIGURATION = new BoxConfiguration(); public static final String BOOK_NAME = BOX_CONFIGURATION.getBookName(); - public static final EventID PARENT_EVENT_ID = toEventID(BOOK_NAME, "id"); + public static final EventID PARENT_EVENT_ID = toEventID(Instant.now(), BOOK_NAME, "id"); private static final ObjectMapper jacksonMapper = jacksonObjectMapper(); diff --git a/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt b/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt index 04ab5131f..4e7c40e35 100644 --- a/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt +++ b/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt @@ -31,11 +31,12 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll +import java.time.Instant typealias ProtoEvent = com.exactpro.th2.common.grpc.Event class TestEvent { - private val parentEventId: EventID = toEventID(BOOK_NAME, "parentEventId")!! + private val parentEventId: EventID = toEventID(Instant.now(), BOOK_NAME, "parentEventId") private val data = EventUtils.createMessageBean("0123456789".repeat(20)) private val dataSize = MAPPER.writeValueAsBytes(listOf(data)).size private val bigData = EventUtils.createMessageBean("0123456789".repeat(30)) @@ -277,7 +278,6 @@ class TestEvent { { assertTrue(hasId()) }, { assertEquals(UNKNOWN_EVENT_NAME, name) }, { assertEquals(UNKNOWN_EVENT_TYPE, type) }, - { assertTrue(hasStartTimestamp()) }, { assertTrue(hasEndTimestamp()) }, { assertEquals(SUCCESS, status) }, { assertEquals(ByteString.copyFrom("[]".toByteArray()), body) }, From 1501289bb551d2d4ca5472c6b20ae74987687f92 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin Date: Tue, 21 Dec 2021 14:08:26 +0400 Subject: [PATCH 015/154] [th2-2876] removed deprecated fields from grpc --- .../exactpro/th2/common/message/MessageFilterUtils.kt | 9 --------- .../com/exactpro/th2/common/message/MessageUtils.kt | 2 +- .../kotlin/com/exactpro/th2/common/TestUtils.kt | 4 ++-- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt index 7aceeb6c3..88b0ef747 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt @@ -40,19 +40,10 @@ import com.exactpro.th2.common.grpc.ValueFilter.KindCase.NULL_VALUE import com.exactpro.th2.common.grpc.ValueFilter.KindCase.SIMPLE_FILTER import com.exactpro.th2.common.value.emptyValueFilter import com.exactpro.th2.common.value.toValueFilter -import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonProperty private val DEFAULT_TIME_PRECISION_REGEX = Regex("(\\d[HMS])(?!\$)") -@Deprecated( - message = "The message type from MessageFilter will be removed in the future", - replaceWith = ReplaceWith( - expression = "rootMessageFilter(messageType)" - ), - level = DeprecationLevel.WARNING -) -fun messageFilter(messageType: String): MessageFilter.Builder = MessageFilter.newBuilder().setMessageType(messageType) fun messageFilter(): MessageFilter.Builder = MessageFilter.newBuilder() fun rootMessageFilter(messageType: String): RootMessageFilter.Builder = RootMessageFilter.newBuilder().setMessageType(messageType) diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt index 7900948a6..bfb5dab56 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt @@ -165,9 +165,9 @@ fun Message.Builder.setMetadata( if (messageType != null) { it.messageType = messageType } - it.timestamp = (timestamp ?: Instant.now()).toTimestamp() if (direction != null || sessionAlias != null) { it.id = MessageID.newBuilder().apply { + this.timestamp = (timestamp ?: Instant.now()).toTimestamp() if (direction != null) { this.direction = direction } diff --git a/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt b/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt index 64dce0946..2047f5ad4 100644 --- a/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt +++ b/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt @@ -56,10 +56,10 @@ fun assertEqualBatches(expected: MessageBatch, actual: MessageBatch, lazyMessage fun assertEqualMessages(expected: Message, actual: Message, lazyMessage: () -> String? = {null}) { val ts = Timestamp.getDefaultInstance() val assertExpected = expected.toBuilder().apply { - metadataBuilder.timestamp = ts + metadataBuilder.idBuilder.timestamp = ts }.build() val assertActual = actual.toBuilder().apply { - metadataBuilder.timestamp = ts + metadataBuilder.idBuilder.timestamp = ts }.build() try { Assertions.assertEquals(assertExpected, assertActual, lazyMessage) From 5ea0cb14d704b0c1514f0a4971b8988a41314371 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Wed, 12 Jan 2022 17:12:41 +0400 Subject: [PATCH 016/154] [th2-2743] added BOOK_NAME_LABEL for check1 (#177) --- src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt b/src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt index 8125e5587..14fb276c2 100644 --- a/src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt +++ b/src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt @@ -47,6 +47,7 @@ const val TH2_TYPE_LABEL = "th2_type" const val EXCHANGE_LABEL = "exchange" const val QUEUE_LABEL = "queue" const val ROUTING_KEY_LABEL = "routing_key" +const val BOOK_NAME_LABEL = "book_name" const val SESSION_ALIAS_LABEL = "session_alias" const val DIRECTION_LABEL = "direction" const val MESSAGE_TYPE_LABEL = "message_type" From 8d7fb2cdcc80be344949de8ee91ee49a3160d253 Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andreydrobynin@users.noreply.github.com> Date: Mon, 24 Jan 2022 15:43:58 +0400 Subject: [PATCH 017/154] [th2-2150] new cradle version (#180) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5c1d555bb..67eb37df5 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-1544972223-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-1739200571-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 5784dc480f1cbb726923bfd86536f0213fd11e6f Mon Sep 17 00:00:00 2001 From: Andrey Drobynin Date: Fri, 11 Feb 2022 22:32:21 +0400 Subject: [PATCH 018/154] fixes after merge --- .../th2/common/schema/factory/CommonFactory.java | 8 +++++--- .../th2/common/schema/factory/FactorySettings.kt | 13 +++++++++---- .../kotlin/com/exactpro/th2/common/TestUtils.kt | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index 8085a69b7..2cbcf45a9 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -145,7 +145,8 @@ protected CommonFactory(Class> messageRout .grpcRouterClass(grpcRouterClass) .variables(environmentVariables) .custom(custom) - .dictionariesDir(dictionariesDir) + .dictionaryTypesDir(dictionaryTypesDir) + .dictionaryAliasesDir(dictionaryAliasesDir) .oldDictionariesDir(oldDictionariesDir) ); } @@ -153,7 +154,8 @@ protected CommonFactory(Class> messageRout public CommonFactory(FactorySettings settings) { super(settings); custom = defaultPathIfNull(settings.getCustom(), CUSTOM_FILE_NAME); - dictionariesDir = defaultPathIfNull(settings.getDictionariesDir(), DICTIONARY_DIR_NAME); + dictionaryTypesDir = defaultPathIfNull(settings.getDictionaryTypesDir(), DICTIONARY_TYPE_DIR_NAME); + dictionaryAliasesDir = defaultPathIfNull(settings.getDictionaryAliasesDir(), DICTIONARY_ALIAS_DIR_NAME); oldDictionariesDir = requireNonNullElse(settings.getOldDictionariesDir(), CONFIG_DEFAULT_PATH); configurationManager = createConfigurationManager(settings); start(); @@ -182,7 +184,7 @@ public CommonFactory(Class> messageRouterP .prometheus(prometheus) .boxConfiguration(boxConfiguration) .custom(custom) - .dictionariesDir(dictionariesDir) + .dictionaryTypesDir(dictionariesDir) ); } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/factory/FactorySettings.kt b/src/main/kotlin/com/exactpro/th2/common/schema/factory/FactorySettings.kt index d73184fc1..47c0c6566 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/factory/FactorySettings.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/factory/FactorySettings.kt @@ -49,7 +49,7 @@ data class FactorySettings @JvmOverloads constructor( var custom: Path? = null, @Deprecated("Will be removed in future releases") var dictionaryTypesDir: Path? = null, var dictionaryAliasesDir: Path? = null, - @Deprecated("Will be removed in future releases") var oldDictionariesDir: Path? = null) { + @Deprecated("Will be removed in future releases") var oldDictionariesDir: Path? = null, var variables: MutableMap = HashMap() ) { fun messageRouterParsedBatchClass(messageRouterParsedBatchClass: Class>): FactorySettings { @@ -132,13 +132,18 @@ data class FactorySettings @JvmOverloads constructor( return this } - fun dictionariesDir(dictionariesDir: Path?): FactorySettings { - this.dictionariesDir = dictionariesDir + fun dictionaryTypesDir(dictionaryTypesDir: Path?): FactorySettings { + this.dictionaryTypesDir = dictionaryTypesDir + return this + } + + fun dictionaryAliasesDir(dictionaryAliasesDir: Path?): FactorySettings { + this.dictionaryAliasesDir = dictionaryAliasesDir return this } fun oldDictionariesDir(oldDictionariesDir: Path?): FactorySettings { - this.oldDictionariesDir = oldDictionariesDir + this.dictionaryTypesDir = oldDictionariesDir return this } diff --git a/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt b/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt index e21bf44c0..f73e4813d 100644 --- a/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt +++ b/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt @@ -194,11 +194,11 @@ fun Message.assertValue(name: String, expected: T? = null): T { } private fun Message.withTimestamp(ts: Timestamp) = toBuilder().apply { - metadataBuilder.timestamp = ts + metadataBuilder.idBuilder.timestamp = ts }.build()!! private fun RawMessage.withTimestamp(ts: Timestamp) = toBuilder().apply { - metadataBuilder.timestamp = ts + metadataBuilder.idBuilder.timestamp = ts }.build()!! private fun AssertionFailedError.rewrap(additional: String) = AssertionFailedError( From 53aae8d5db6aefdd89cf73203afc5e9cc0d20fff Mon Sep 17 00:00:00 2001 From: Andrey Drobynin <89458084+andrey-drobynin@users.noreply.github.com> Date: Mon, 28 Feb 2022 17:02:12 +0400 Subject: [PATCH 019/154] missed keyspace (#158) * added prepareStorage to README * added missed keyspace set to cradle --- .../th2/common/schema/factory/AbstractCommonFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 1d218dbad..41390a427 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -522,6 +522,7 @@ public CradleManager getCradleManager() { DEFAULT_CONSISTENCY_LEVEL, DEFAULT_CONSISTENCY_LEVEL ); + cassandraStorageSettings.setCradleInfoKeyspace(confidentialConfiguration.getKeyspace()); if (nonConfidentialConfiguration.getPageSize() > 0) { cassandraStorageSettings.setResultPageSize(nonConfidentialConfiguration.getPageSize()); } From 4104348d57e8c1cdf56e105d120984f1a35c7229 Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Fri, 8 Apr 2022 16:13:15 +0400 Subject: [PATCH 020/154] [TH2-3528] Updated cradle version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 67eb37df5..a36bb8944 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-1739200571-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-2096569469-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 18048ccee44cdb5fe2b2f0ed612ad63a4972440c Mon Sep 17 00:00:00 2001 From: nchur17 Date: Mon, 2 May 2022 16:18:04 +0400 Subject: [PATCH 021/154] cradle version updated --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a36bb8944..6f3aed366 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-2096569469-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-2257485997-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 12b6893f09062742efd94abd23a1a13ae71fed1b Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Wed, 4 May 2022 17:35:32 +0400 Subject: [PATCH 022/154] [TH2-3591] Updated cradle version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6f3aed366..7ee504ef2 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-2257485997-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-2269130595-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 648ff825f653c31e2fa6c83b7523e8e40c1d26d4 Mon Sep 17 00:00:00 2001 From: georgiano Date: Tue, 10 May 2022 15:19:43 +0400 Subject: [PATCH 023/154] [H2-3578] updated cradle library with groups --- build.gradle | 6 +++--- gradle.properties | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index fdda3082a..e3c283fd9 100644 --- a/build.gradle +++ b/build.gradle @@ -30,8 +30,8 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '3.1.1' - junitVersion = '5.7.2' + cradleVersion = '3.2.0-ver_3_1_dev-2300230098-SNAPSHOT' + junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } @@ -168,7 +168,7 @@ tasks.register('integrationTest', Test) { dependencies { api platform('com.exactpro.th2:bom:3.1.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'com.exactpro.th2:grpc-common:3.10.0' + api 'com.exactpro.th2:grpc-common:3.11.0' implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.2.1' diff --git a/gradle.properties b/gradle.properties index 4dfb01f80..54dad2d31 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.37.1 +release_version=3.38.0 description = 'th2 common library (Java)' From d7ade9a57d085f130f484d5843724a2f42c52cde Mon Sep 17 00:00:00 2001 From: georgiano Date: Tue, 17 May 2022 16:26:56 +0400 Subject: [PATCH 024/154] [TH2-3699] updated with grpc-common supporting session groups --- build.gradle | 2 +- .../th2/common/builder/MessageEventIdBuildersTest.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 7ee504ef2..93027f877 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,7 @@ test { dependencies { api platform('com.exactpro.th2:bom:3.0.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'com.exactpro.th2:grpc-common:4.0.0-th2-2150-books-pages-1606698156-SNAPSHOT' + api 'com.exactpro.th2:grpc-common:4.0.0-th2-2150-books-pages-2338290872-SNAPSHOT' implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.1.12' diff --git a/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java b/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java index 354fa6363..39a281674 100644 --- a/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java +++ b/src/test/java/com/exactpro/th2/common/builder/MessageEventIdBuildersTest.java @@ -32,6 +32,7 @@ public class MessageEventIdBuildersTest { private static final String CONFIG_DIRECTORY = "src/test/resources/test_message_event_id_builders"; private static final String CUSTOM_BOOK = "custom_book"; private static final String DEFAULT_ALIAS = "alias"; + private static final String DEFAULT_GROUP = "group"; private static final Direction DEFAULT_DIRECTION = Direction.FIRST; private static final int DEFAULT_SEQUENCE = 1; private static final int DEFAULT_SUBSEQUENCE = 2; @@ -67,7 +68,8 @@ private void assertIds(String bookName, MessageID.Builder messageIdBuilder, Even assertEquals( "{\n" + " \"connectionId\": {\n" + - " \"sessionAlias\": \"" + DEFAULT_ALIAS + "\"\n" + + " \"sessionAlias\": \"" + DEFAULT_ALIAS + "\",\n" + + " \"sessionGroup\": \"" + DEFAULT_GROUP + "\"\n" + " },\n" + " \"direction\": \"" + DEFAULT_DIRECTION.name() + "\",\n" + " \"sequence\": \"" + DEFAULT_SEQUENCE + "\",\n" + @@ -88,7 +90,7 @@ private void assertIds(String bookName, MessageID.Builder messageIdBuilder, Even private MessageID.Builder defaultMessageIdBuilder() { return commonFactory.newMessageIDBuilder() - .setConnectionId(ConnectionID.newBuilder().setSessionAlias(DEFAULT_ALIAS)) + .setConnectionId(ConnectionID.newBuilder().setSessionAlias(DEFAULT_ALIAS).setSessionGroup(DEFAULT_GROUP)) .setDirection(DEFAULT_DIRECTION) .setSequence(DEFAULT_SEQUENCE) .addSubsequence(DEFAULT_SUBSEQUENCE); From 76b1333b70d0b01f34306f5df69e0ad1b635b12e Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Fri, 20 May 2022 10:17:54 +0400 Subject: [PATCH 025/154] merged with master --- README.md | 34 ++- build.gradle | 29 ++- .../event/bean/builder/CollectionBuilder.java | 5 +- .../schema/event/EventBatchSubscriber.java | 8 +- .../schema/factory/AbstractCommonFactory.java | 10 +- .../common/schema/factory/CommonFactory.java | 46 ++-- .../strategy/impl/AbstractFilterStrategy.java | 32 +-- .../GrpcEndpointConfiguration.java | 0 .../grpc/router/impl/DefaultGrpcRouter.java | 27 ++- .../grpc/router/impl/DefaultStubStorage.java | 58 ++++- .../schema/message/MessageListener.java | 4 +- .../common/schema/message/MessageRouter.java | 39 +++- .../schema/message/MessageSubscriber.java | 4 +- .../rabbitmq/AbstractRabbitSubscriber.java | 47 ++-- .../connection/ConnectionManager.java | 216 +++++++++++++----- .../strategy/route/RoutingStrategy.java | 2 +- .../route/RoutingStrategyFactory.java | 1 + .../schema/strategy/route/StrategyName.java | 1 + .../configuration/ConfigurationManager.kt | 2 +- .../filter/strategy/impl/FieldValueChecker.kt | 30 +++ .../grpc/configuration/GrpcConfiguration.kt | 12 +- .../message/ConfirmationMessageListener.kt | 62 +++++ .../message/ManualAckDeliveryCallback.kt | 36 +++ .../message/impl/OnlyOnceConfirmation.kt | 44 ++++ .../AbstractGroupBatchAdapterRouter.kt | 10 +- .../impl/rabbitmq/AbstractRabbitRouter.kt | 37 ++- .../configuration/RabbitMQConfiguration.kt | 12 +- .../RabbitMessageGroupBatchSubscriber.kt | 12 +- .../NotificationEventBatchRouter.kt | 5 +- .../NotificationEventBatchSubscriber.kt | 35 +-- src/main/resources/log4j.properties | 8 +- .../th2/common/annotations/IntegrationTest.kt | 24 ++ .../common/schema/TestJsonConfiguration.kt | 7 +- .../TestConfirmationMessageListenerWrapper.kt | 50 ++++ .../connection/TestConnectionManager.kt | 105 +++++++++ .../TestMessageConverterLambdaDelegate.kt | 5 +- 36 files changed, 842 insertions(+), 217 deletions(-) delete mode 100644 src/main/java/com/exactpro/th2/common/schema/grpc/configuration/GrpcEndpointConfiguration.java create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/FieldValueChecker.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/TestConnectionManager.kt diff --git a/README.md b/README.md index 93848ef6d..34929d777 100644 --- a/README.md +++ b/README.md @@ -301,11 +301,43 @@ dependencies { + Added `prepareStorage` property to `cradle.json` + `com.exactpro.th2.common.event.Event.toProto...()` by `parentEventId`/`bookName`/`(bookName + scope)` -### 3.33.0 +### 3.37.2 + ++ Corrected logging for configuration files + +### 3.37.1 + ++ Fixed: + + When creating the `CommonFactory` from k8s the logging configuration wouldn't be downloaded + +### 3.37.0 + ++ Added support for gRPC pins filters + +### 3.36.0 + +* Cradle version was updated from `2.20.2` to `3.1.1`. + **Please, note, that migration is required for `3.1.1` usage**. +* New parameter `prepareStorage` is added to the `cradle_manager.json`. + It allows enabling/disabling Cradle schema initialization. + +### 3.35.0 + +* Included dependency to the io.prometheus:simpleclient_log4j:0.9.0 + +### 3.34.0 + Added ability to read dictionaries by aliases and as group of all available aliases + New methods for api: loadDictionary(String), getDictionaryAliases(), loadSingleDictionary() +### 3.33.0 + +#### Added: + ++ Methods for subscription with manual acknowledgement + (if the **prefetch count** is requested and no messages are acknowledged the reading from the queue will be suspended). + Please, note that only one subscriber with manual acknowledgement can be subscribed to a queue + ### 3.32.1 + Fixed: gRPC router didn't shut down underlying Netty's EventLoopGroup and ExecutorService diff --git a/build.gradle b/build.gradle index 93027f877..7aff2d717 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ buildscript { } ext { - kotlin_version = "1.5.30" + kotlin_version = "1.5.32" } dependencies { @@ -153,16 +153,25 @@ signing { } test { - useJUnitPlatform() + useJUnitPlatform { + excludeTags('integration-test') + } +} + +tasks.register('integrationTest', Test) { + group = 'verification' + useJUnitPlatform { + includeTags('integration-test') + } } dependencies { - api platform('com.exactpro.th2:bom:3.0.0') + api platform('com.exactpro.th2:bom:3.1.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" api 'com.exactpro.th2:grpc-common:4.0.0-th2-2150-books-pages-2338290872-SNAPSHOT' implementation 'com.google.protobuf:protobuf-java-util' - implementation 'com.exactpro.th2:grpc-service-generator:3.1.12' + implementation 'com.exactpro.th2:grpc-service-generator:3.2.1' implementation "com.exactpro.th2:cradle-cassandra:${cradleVersion}" //FIXME: Add these dependencies as api to grpc-... artifacts @@ -184,6 +193,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-annotations" implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" + // TODO: need to update jackson kotlin module to get rid of kotlin-reflect 1.3.72 implementation "com.fasterxml.jackson.module:jackson-module-kotlin" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" @@ -191,7 +201,7 @@ dependencies { implementation 'com.fasterxml.uuid:java-uuid-generator:4.0.1' - implementation 'io.github.microutils:kotlin-logging:2.0.11' + implementation 'io.github.microutils:kotlin-logging:2.1.21' implementation "org.slf4j:slf4j-log4j12" implementation "org.slf4j:slf4j-api" @@ -199,11 +209,14 @@ dependencies { implementation 'io.prometheus:simpleclient' implementation 'io.prometheus:simpleclient_hotspot' implementation 'io.prometheus:simpleclient_httpserver' + implementation 'io.prometheus:simpleclient_log4j:0.9.0' implementation 'io.fabric8:kubernetes-client:4.13.0' testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" testImplementation 'org.mockito.kotlin:mockito-kotlin:4.0.0' + testImplementation "org.testcontainers:testcontainers:1.16.2" + testImplementation "org.testcontainers:rabbitmq:1.16.2" testFixturesImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit5', version: kotlin_version testFixturesImplementation 'org.junit.jupiter:junit-jupiter:5.8.1' @@ -227,11 +240,7 @@ sourceSets { main.kotlin.srcDirs += "src/main/kotlin" } -compileKotlin { - kotlinOptions.jvmTarget = "11" -} - -compileTestKotlin { +tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { kotlinOptions.jvmTarget = "11" } diff --git a/src/main/java/com/exactpro/th2/common/event/bean/builder/CollectionBuilder.java b/src/main/java/com/exactpro/th2/common/event/bean/builder/CollectionBuilder.java index d13c1a97e..74163ee65 100644 --- a/src/main/java/com/exactpro/th2/common/event/bean/builder/CollectionBuilder.java +++ b/src/main/java/com/exactpro/th2/common/event/bean/builder/CollectionBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,8 @@ */ package com.exactpro.th2.common.event.bean.builder; -import com.exactpro.th2.common.event.bean.*; +import com.exactpro.th2.common.event.bean.Collection; +import com.exactpro.th2.common.event.bean.TreeTableEntry; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java index a5986032d..b99619e54 100644 --- a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import com.exactpro.th2.common.schema.message.FilterFunction; import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.rabbitmq.client.Delivery; import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_PIN_LABEL; @@ -69,10 +70,11 @@ protected EventBatch filter(EventBatch eventBatch) throws Exception { } @Override - protected void handle(String consumeTag, Delivery delivery, EventBatch value) { + protected void handle(String consumeTag, Delivery delivery, EventBatch value, + Confirmation confirmation) { EVENT_SUBSCRIBE_TOTAL .labels(th2Pin) .inc(value.getEventsCount()); - super.handle(consumeTag, delivery, value); + super.handle(consumeTag, delivery, value, confirmation); } } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 41390a427..554cbb5a1 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -58,6 +58,7 @@ import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fasterxml.jackson.module.kotlin.KotlinModule; import io.prometheus.client.exporter.HTTPServer; import io.prometheus.client.hotspot.DefaultExports; @@ -95,7 +96,6 @@ import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_CONSISTENCY_LEVEL; import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_TIMEOUT; -import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.StringUtils.defaultIfBlank; @@ -119,9 +119,11 @@ public abstract class AbstractCommonFactory implements AutoCloseable { protected static final ObjectMapper MAPPER = new ObjectMapper(); static { - MAPPER.registerModule(new KotlinModule()); - - MAPPER.registerModule(new RoutingStrategyModule(MAPPER)); + MAPPER.registerModules( + new KotlinModule(), + new RoutingStrategyModule(MAPPER), + new JavaTimeModule() + ); } private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCommonFactory.class); diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index 2cbcf45a9..0cb9c0e2c 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -113,6 +113,9 @@ public class CommonFactory extends AbstractCommonFactory { private static final String KEY_CASSANDRA_PASS = "CASSANDRA_PASS"; private static final String GENERATED_CONFIG_DIR_NAME = "generated_configs"; + private static final String RABBIT_MQ_EXTERNAL_APP_CONFIG_MAP = "rabbit-mq-external-app-config"; + private static final String CRADLE_EXTERNAL_MAP = "cradle-external"; + private static final String LOGGING_CONFIG_MAP = "logging-config"; private final Path custom; private final Path dictionaryTypesDir; @@ -403,13 +406,6 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam * @return CommonFactory with set path */ public static CommonFactory createFromKubernetes(String namespace, String boxName, @Nullable String contextName, @NotNull Map dictionaries) { - Resource boxConfigMapResource; - Resource rabbitMqConfigMapResource; - Resource cradleConfigMapResource; - - ConfigMap boxConfigMap; - ConfigMap rabbitMqConfigMap; - ConfigMap cradleConfigMap; Path configPath = Path.of(System.getProperty("user.dir"), GENERATED_CONFIG_DIR_NAME); @@ -442,20 +438,26 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam var configMaps = client.configMaps(); - boxConfigMapResource = configMaps.inNamespace(namespace).withName(boxName + "-app-config"); - rabbitMqConfigMapResource = configMaps.inNamespace(namespace).withName("rabbit-mq-external-app-config"); - cradleConfigMapResource = configMaps.inNamespace(namespace).withName("cradle-external"); + Resource boxConfigMapResource = configMaps.inNamespace(namespace).withName(boxName + "-app-config"); - if (boxConfigMapResource.get() == null) + if (boxConfigMapResource.get() == null) { throw new IllegalArgumentException("Failed to find config maps by boxName " + boxName); + } + Resource rabbitMqConfigMapResource = configMaps.inNamespace(namespace).withName(RABBIT_MQ_EXTERNAL_APP_CONFIG_MAP); + Resource cradleConfigMapResource = configMaps.inNamespace(namespace).withName(CRADLE_EXTERNAL_MAP); + Resource loggingConfigMapResource = configMaps.inNamespace(namespace).withName(LOGGING_CONFIG_MAP); - boxConfigMap = boxConfigMapResource.require(); - rabbitMqConfigMap = rabbitMqConfigMapResource.require(); - cradleConfigMap = cradleConfigMapResource.require(); + ConfigMap boxConfigMap = boxConfigMapResource.require(); + ConfigMap rabbitMqConfigMap = rabbitMqConfigMapResource.require(); + ConfigMap cradleConfigMap = cradleConfigMapResource.require(); + @Nullable ConfigMap loggingConfigMap = loggingConfigMapResource.get(); Map boxData = boxConfigMap.getData(); Map rabbitMqData = rabbitMqConfigMap.getData(); Map cradleConfigData = cradleConfigMap.getData(); + @Nullable String loggingData = boxData.getOrDefault(LOG4J_PROPERTIES_NAME, + loggingConfigMap == null ? null : loggingConfigMap.getData().get(LOG4J_PROPERTIES_NAME) + ); File generatedConfigsDirFile = configPath.toFile(); @@ -466,6 +468,14 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam } if (generatedConfigsDirFile.exists()) { + BoxConfiguration box = new BoxConfiguration(); + box.setBoxName(boxName); + + if (loggingData != null) { + writeFile(configPath.resolve(LOG4J_PROPERTIES_NAME), loggingData); + configureLogger(configPath.toString()); + } + settings.setRabbitMQ(writeFile(configPath, RABBIT_MQ_FILE_NAME, rabbitMqData)); settings.setRouterMQ(writeFile(configPath, ROUTER_MQ_FILE_NAME, boxData)); settings.setConnectionManagerSettings(writeFile(configPath, CONNECTION_MANAGER_CONF_FILE_NAME, boxData)); @@ -482,12 +492,10 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam String boxConfig = boxData.get(BOX_FILE_NAME); - if (boxConfig != null) { - writeFile(boxConfigurationPath, boxConfig); - } else { - BoxConfiguration box = new BoxConfiguration(); - box.setBoxName(boxName); + if (boxConfig == null) { writeToJson(boxConfigurationPath, box); + } else { + writeFile(boxConfigurationPath, boxConfig); } writeDictionaries(boxName, configPath, dictionaryTypePath, dictionaries, configMaps.list()); diff --git a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java index 4076e740a..fee14236e 100644 --- a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java +++ b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -21,14 +21,10 @@ import com.google.protobuf.Message; import org.apache.commons.collections4.MultiMapUtils; import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Objects; - public abstract class AbstractFilterStrategy implements FilterStrategy { @@ -55,33 +51,11 @@ public boolean verify(T message, List routerFilters) { protected abstract Map getFields(T message); - private boolean checkValues(Map messageFields, MultiValuedMap fieldFilters) { return fieldFilters.isEmpty() || fieldFilters.keys().stream().anyMatch(fieldName -> { String messageValue = messageFields.get(fieldName); Collection filters = fieldFilters.get(fieldName); - return !filters.isEmpty() && filters.stream().allMatch(filter -> checkValue(messageValue, filter)); + return !filters.isEmpty() && filters.stream().allMatch(filter -> FieldValueChecker.checkFieldValue(filter, messageValue)); }); } - - private boolean checkValue(String value, FieldFilterConfiguration filterConfiguration) { - var valueInConf = filterConfiguration.getExpectedValue(); - - // FIXME: Change switch to switch-expression after upping java version - switch (filterConfiguration.getOperation()) { - case EQUAL: - return Objects.equals(value, valueInConf); - case NOT_EQUAL: - return !Objects.equals(value, valueInConf); - case EMPTY: - return StringUtils.isEmpty(value); - case NOT_EMPTY: - return StringUtils.isNotEmpty(value); - case WILDCARD: - return FilenameUtils.wildcardMatch(value, valueInConf); - default: - return false; - } - } - -} +} \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/configuration/GrpcEndpointConfiguration.java b/src/main/java/com/exactpro/th2/common/schema/grpc/configuration/GrpcEndpointConfiguration.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java index 2d79d6a62..7c544f3f2 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -149,7 +149,8 @@ protected AbstractStub getGrpcStubToSend(Class proxyService, Message messa } protected AbstractStub getStubInstanceOrCreate(Class proxyService, Class stubClass, Message message) { - var serviceConfig = getServiceConfig(proxyService); + // FIXME: Add gRPC pin filters if needed + var serviceConfig = getServiceConfig(proxyService).get(0); String endpointName = serviceConfig.getStrategy().getEndpoint(message); @@ -158,8 +159,8 @@ protected AbstractStub getStubInstanceOrCreate(Class proxyService) { - return configuration.getServices().values().stream() + protected List getServiceConfig(Class proxyService) { + final var result = configuration.getServices().values().stream() .filter(sConfig -> { String proxyClassName = proxyService.getName(); @@ -170,9 +171,13 @@ protected GrpcServiceConfiguration getServiceConfig(Class proxyService) { return sConfig.getServiceClass().getName().equals(proxyClassName); }) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No services matching the provided " + - "class were found in the configuration: " + proxyService.getName())); + .collect(Collectors.toList()); + if (result.isEmpty()) { + throw new IllegalStateException("No services matching the provided class were found in the configuration: " + + proxyService.getName()); + } + + return result; } protected Channel getOrCreateChannel(String endpointName, GrpcServiceConfiguration serviceConfig) { @@ -191,9 +196,9 @@ protected Channel getOrCreateChannel(String endpointName, GrpcServiceConfigurati @SuppressWarnings("rawtypes") protected AbstractStub createStubInstance(Class stubClass, Channel channel) { try { - var constr = stubClass.getDeclaredConstructor(Channel.class, CallOptions.class); - constr.setAccessible(true); - return constr.newInstance(channel, CallOptions.DEFAULT); + var constructor = stubClass.getDeclaredConstructor(Channel.class, CallOptions.class); + constructor.setAccessible(true); + return constructor.newInstance(channel, CallOptions.DEFAULT); } catch (NoSuchMethodException e) { throw new InitGrpcRouterException("Could not find constructor " + "'(Channel,CallOptions)' in the provided stub class: " + stubClass, e); @@ -201,4 +206,4 @@ protected AbstractStub createStubInstance(Class stub throw new InitGrpcRouterException("Something went wrong while creating stub instance: " + stubClass, e); } } -} +} \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java index 4e88bc29e..dbfd53a44 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,14 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.exactpro.th2.common.schema.grpc.router.impl; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; import javax.annotation.concurrent.ThreadSafe; +import com.exactpro.th2.common.schema.filter.strategy.impl.FieldValueChecker; +import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration; import org.jetbrains.annotations.NotNull; import com.exactpro.th2.common.schema.grpc.configuration.GrpcEndpointConfiguration; @@ -35,19 +41,47 @@ @ThreadSafe public class DefaultStubStorage> implements StubStorage { - private final GrpcServiceConfiguration serviceConfiguration; - private final Map stubs = new ConcurrentHashMap<>(); + private static class ServiceHolder { + GrpcServiceConfiguration serviceConfig; + Map stubs = new ConcurrentHashMap<>(); + + ServiceHolder(GrpcServiceConfiguration serviceConfig) { + this.serviceConfig = serviceConfig; + } + } + + private final List> services; - public DefaultStubStorage(@NotNull GrpcServiceConfiguration serviceConfiguration) { - this.serviceConfiguration = Objects.requireNonNull(serviceConfiguration, "Service configuration can not be null"); + public DefaultStubStorage(@NotNull List serviceConfigurations) { + services = new ArrayList<>(serviceConfigurations.size()); + for (GrpcServiceConfiguration config: serviceConfigurations) { + services.add(new ServiceHolder<>(config)); + } } @NotNull @Override - public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory stubFactory) { - String endpointLabel = serviceConfiguration.getStrategy().getEndpoint(message); - return stubs.computeIfAbsent(endpointLabel, key -> { - GrpcEndpointConfiguration endpoint = serviceConfiguration.getEndpoints().get(key); + public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory stubFactory, @NotNull Map properties) { + + final var matchingServices = services.stream() + .filter(service -> service.serviceConfig.getFilters().isEmpty() + || service.serviceConfig.getFilters().stream().anyMatch(it -> isAllPropertiesMatch(it.getProperties(), properties))) + .limit(2) + .collect(Collectors.toList()); + + if(matchingServices.isEmpty()) { + throw new IllegalStateException("No gRPC pin matches the provided properties: " + properties); + } + + if(matchingServices.size() > 1) { + throw new IllegalStateException("More than one gRPC pins match the provided properties: " + properties); + } + + final var service = matchingServices.get(0); + final var endpointLabel = service.serviceConfig.getStrategy().getEndpoint(message); + + return service.stubs.computeIfAbsent(endpointLabel, key -> { + GrpcEndpointConfiguration endpoint = service.serviceConfig.getEndpoints().get(key); if (Objects.isNull(endpoint)) { throw new IllegalStateException("No endpoint in the configuration " + @@ -57,4 +91,8 @@ public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory return stubFactory.newStub(ManagedChannelBuilder.forAddress(endpoint.getHost(), endpoint.getPort()).usePlaintext().build(), CallOptions.DEFAULT); }); } -} + + private boolean isAllPropertiesMatch(List filterProp, Map properties) { + return filterProp.stream().allMatch(it -> FieldValueChecker.checkFieldValue(it, properties.get(it.getFieldName()))); + } +} \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java index 0b1baefe1..6cbc0570d 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ */ public interface MessageListener { - void handler(String consumerTag, T message) throws Exception; + void handle(String consumerTag, T message) throws Exception; default void onClose() {} diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index 27113be53..a21672179 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -22,7 +22,6 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.util.Objects; @@ -57,12 +56,11 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< /** * Listen ONE RabbitMQ queue by intersection schemas queues attributes - * @param queueAttr queues attributes * @param callback listener + * @param queueAttr queues attributes * @throws IllegalStateException when more than 1 queue is found * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue */ - @Nullable SubscriberMonitor subscribe(MessageListener callback, String... queueAttr); /** @@ -70,7 +68,6 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< * @param callback listener * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue */ - @Nullable default SubscriberMonitor subscribeAll(MessageListener callback) { return subscribeAll(callback, QueueAttribute.SUBSCRIBE.toString()); } @@ -81,9 +78,41 @@ default SubscriberMonitor subscribeAll(MessageListener callback) { * @param queueAttr queues attributes * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue */ - @Nullable SubscriberMonitor subscribeAll(MessageListener callback, String... queueAttr); + /** + * Listen ONE RabbitMQ queue by intersection schemas queues attributes + * @param queueAttr queues attributes + * @param callback listener with manual confirmation + * @throws IllegalStateException when more than 1 queue is found + * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + */ + default SubscriberMonitor subscribeWithManualAck(ConfirmationMessageListener callback, String... queueAttr) { + // TODO: probably should not have default implementation + throw new UnsupportedOperationException("The subscription with manual confirmation is not supported"); + } + + /** + * Listen ALL RabbitMQ queues in configurations + * @param callback listener with manual confirmation + * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + */ + default SubscriberMonitor subscribeAllWithManualAck(ConfirmationMessageListener callback) { + // TODO: probably should not have default implementation + return subscribeAllWithManualAck(callback, QueueAttribute.SUBSCRIBE.toString()); + } + + /** + * Listen SOME RabbitMQ queues by intersection schemas queues attributes + * @param callback listener with manual confirmation + * @param queueAttr queues attributes + * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + */ + default SubscriberMonitor subscribeAllWithManualAck(ConfirmationMessageListener callback, String... queueAttr) { + // TODO: probably should not have default implementation + throw new UnsupportedOperationException("The subscription with manual confirmation is not supported"); + } + /** * Send message to SOME RabbitMQ queues which match the filter for this message * @param message diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java index fb0357134..0bc1ba7da 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -37,5 +37,5 @@ public interface MessageSubscriber extends AutoCloseable { void start() throws Exception; - void addListener(MessageListener messageListener); + void addListener(ConfirmationMessageListener messageListener); } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index b6c8d3c3e..a76e3818e 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -16,8 +16,9 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq; import com.exactpro.th2.common.metrics.HealthMetrics; +import com.exactpro.th2.common.schema.message.ConfirmationMessageListener; import com.exactpro.th2.common.schema.message.FilterFunction; -import com.exactpro.th2.common.schema.message.MessageListener; +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.exactpro.th2.common.schema.message.MessageSubscriber; import com.exactpro.th2.common.schema.message.SubscriberMonitor; import com.exactpro.th2.common.schema.message.configuration.RouterFilter; @@ -26,9 +27,6 @@ import com.google.protobuf.Message; import com.rabbitmq.client.Delivery; -import static com.exactpro.th2.common.metrics.CommonMetrics.QUEUE_LABEL; -import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_PIN_LABEL; -import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_TYPE_LABEL; import io.prometheus.client.Counter; import io.prometheus.client.Histogram; import io.prometheus.client.Histogram.Timer; @@ -42,9 +40,13 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import static com.exactpro.th2.common.metrics.CommonMetrics.DEFAULT_BUCKETS; +import static com.exactpro.th2.common.metrics.CommonMetrics.QUEUE_LABEL; +import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_PIN_LABEL; +import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_TYPE_LABEL; import static java.util.Objects.requireNonNull; public abstract class AbstractRabbitSubscriber implements MessageSubscriber { @@ -67,12 +69,13 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber> listeners = new CopyOnWriteArrayList<>(); + private final List> listeners = new CopyOnWriteArrayList<>(); private final String queue; private final AtomicReference connectionManager = new AtomicReference<>(); private final AtomicReference consumerMonitor = new AtomicReference<>(); private final AtomicReference filterFunc = new AtomicReference<>(); private final String th2Type; + private final AtomicBoolean hasManualSubscriber = new AtomicBoolean(); private final HealthMetrics healthMetrics = new HealthMetrics(this); @@ -115,7 +118,7 @@ public void start() throws Exception { try { monitor = connectionManager.basicConsume( queue, - (consumeTag, delivery) -> { + (consumeTag, delivery, confirmation) -> { Timer processTimer = MESSAGE_PROCESS_DURATION_SECONDS .labels(th2Pin, th2Type, queue) .startTimer(); @@ -135,7 +138,7 @@ public void start() throws Exception { e ); } - handle(consumeTag, delivery, value); + handle(consumeTag, delivery, value, confirmation); } finally { processTimer.observeDuration(); } @@ -156,7 +159,13 @@ public void start() throws Exception { } @Override - public void addListener(MessageListener messageListener) { + public void addListener(ConfirmationMessageListener messageListener) { + if (ConfirmationMessageListener.isManual(messageListener)) { + if (!hasManualSubscriber.compareAndSet(false, true)) { + throw new IllegalStateException("cannot subscribe listener " + messageListener + + " because only one listener with manual confirmation is allowed per queue"); + } + } listeners.add(messageListener); } @@ -172,7 +181,7 @@ public void close() throws Exception { monitor.unsubscribe(); } - listeners.forEach(MessageListener::onClose); + listeners.forEach(ConfirmationMessageListener::onClose); listeners.clear(); } @@ -194,14 +203,15 @@ protected boolean callFilterFunction(Message message, List "Received value from " + routingKey + " is null"); if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Received message: {}", toShortTraceString(value)); + LOGGER.trace("Received message from {}: {}", routingKey, toShortTraceString(value)); } else if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Received message: {}", toShortDebugString(value)); + LOGGER.debug("Received message from {}: {}", routingKey, toShortDebugString(value)); } var filteredValue = filter(value); @@ -211,13 +221,20 @@ protected void handle(String consumeTag, Delivery delivery, T value) { return; } - for (MessageListener listener : listeners) { + boolean hasManualConfirmation = false; + for (ConfirmationMessageListener listener : listeners) { try { - listener.handler(consumeTag, filteredValue); + listener.handle(consumeTag, filteredValue, confirmation); + if (!hasManualConfirmation) { + hasManualConfirmation = ConfirmationMessageListener.isManual(listener); + } } catch (Exception listenerExc) { LOGGER.warn("Message listener from class '{}' threw exception", listener.getClass(), listenerExc); } } + if (!hasManualConfirmation) { + confirmation.confirm(); + } } catch (Exception e) { LOGGER.error("Can not parse value from delivery for: {}", consumeTag, e); } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java index c434ce342..f9eeba828 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -14,8 +14,41 @@ */ package com.exactpro.th2.common.schema.message.impl.rabbitmq.connection; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +import javax.annotation.concurrent.GuardedBy; + +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.exactpro.th2.common.metrics.HealthMetrics; +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback; +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.exactpro.th2.common.schema.message.SubscriberMonitor; +import com.exactpro.th2.common.schema.message.impl.OnlyOnceConfirmation; import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration; import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -26,37 +59,12 @@ import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; import com.rabbitmq.client.Consumer; -import com.rabbitmq.client.DeliverCallback; +import com.rabbitmq.client.Envelope; import com.rabbitmq.client.ExceptionHandler; import com.rabbitmq.client.Recoverable; import com.rabbitmq.client.RecoveryListener; import com.rabbitmq.client.ShutdownNotifier; import com.rabbitmq.client.TopologyRecoveryException; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.commons.lang3.builder.ToStringBuilder; -import org.apache.commons.lang3.builder.ToStringStyle; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.BiConsumer; -import java.util.function.Supplier; public class ConnectionManager implements AutoCloseable { public static final String EMPTY_ROUTING_KEY = ""; @@ -69,8 +77,9 @@ public class ConnectionManager implements AutoCloseable { private final ConnectionManagerConfiguration configuration; private final String subscriberName; private final AtomicInteger nextSubscriberId = new AtomicInteger(1); - private final ExecutorService sharedExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() - .setNameFormat("rabbitmq-shared-pool-%d") + private final ExecutorService sharedExecutor; + private final ScheduledExecutorService channelChecker = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder() + .setNameFormat("channel-checker-%d") .build()); private final HealthMetrics metrics = new HealthMetrics(this); @@ -211,6 +220,9 @@ private void turnOffReadiness(Throwable exception){ return recoveryDelay; } ); + sharedExecutor = Executors.newFixedThreadPool(configuration.getWorkingThreads(), new ThreadFactoryBuilder() + .setNameFormat("rabbitmq-shared-pool-%d") + .build()); factory.setSharedExecutor(sharedExecutor); try { @@ -251,9 +263,12 @@ public boolean isOpen() { @Override public void close() { if (connectionIsClosed.getAndSet(true)) { + LOGGER.info("Connection manager already closed"); return; } + LOGGER.info("Closing connection manager"); + int closeTimeout = configuration.getConnectionCloseTimeout(); if (connection.isOpen()) { try { @@ -265,35 +280,62 @@ public void close() { } } - shutdownSharedExecutor(closeTimeout); + shutdownExecutor(sharedExecutor, closeTimeout, "rabbit-shared"); + shutdownExecutor(channelChecker, closeTimeout, "channel-checker"); } public void basicPublish(String exchange, String routingKey, BasicProperties props, byte[] body) throws IOException { ChannelHolder holder = getChannelFor(PinId.forRoutingKey(routingKey)); - holder.withLock(channel -> { - channel.basicPublish(exchange, routingKey, props, body); - }); + holder.withLock(channel -> channel.basicPublish(exchange, routingKey, props, body)); } - public SubscriberMonitor basicConsume(String queue, DeliverCallback deliverCallback, CancelCallback cancelCallback) throws IOException { + public SubscriberMonitor basicConsume(String queue, ManualAckDeliveryCallback deliverCallback, CancelCallback cancelCallback) throws IOException { ChannelHolder holder = getChannelFor(PinId.forQueue(queue)); - String tag = holder.mapWithLock(channel -> { - return channel.basicConsume(queue, false, subscriberName + "_" + nextSubscriberId.getAndIncrement(), (tagTmp, delivery) -> { + String tag = holder.mapWithLock(channel -> + channel.basicConsume(queue, false, subscriberName + "_" + nextSubscriberId.getAndIncrement(), (tagTmp, delivery) -> { try { - try { - deliverCallback.handle(tagTmp, delivery); - } finally { - holder.withLock(ch -> basicAck(ch, delivery.getEnvelope().getDeliveryTag())); - } + Envelope envelope = delivery.getEnvelope(); + long deliveryTag = envelope.getDeliveryTag(); + String routingKey = envelope.getRoutingKey(); + LOGGER.trace("Received delivery {} from queue={} routing_key={}", deliveryTag, queue, routingKey); + + Confirmation confirmation = OnlyOnceConfirmation.wrap("from " + routingKey + " to " + queue, () -> holder.withLock(ch -> { + try { + basicAck(ch, deliveryTag); + } finally { + holder.release(() -> metrics.getReadinessMonitor().enable()); + } + })); + + holder.withLock(() -> holder.acquireAndSubmitCheck(() -> + channelChecker.schedule(() -> { + holder.withLock(() -> { + LOGGER.warn("The confirmation for delivery {} in queue={} routing_key={} was not invoked within the specified delay", + deliveryTag, queue, routingKey); + if (holder.reachedPendingLimit()) { + metrics.getReadinessMonitor().disable(); + } + }); + return false; // to cast to Callable + }, configuration.getConfirmationTimeout().toMillis(), TimeUnit.MILLISECONDS) + )); + deliverCallback.handle(tagTmp, delivery, confirmation); } catch (IOException | RuntimeException e) { LOGGER.error("Cannot handle delivery for tag {}: {}", tagTmp, e.getMessage(), e); } - }, cancelCallback); - }); + }, cancelCallback)); return new RabbitMqSubscriberMonitor(holder, tag, this::basicCancel); } + boolean isReady() { + return metrics.getReadinessMonitor().isEnabled(); + } + + boolean isAlive() { + return metrics.getLivenessMonitor().isEnabled(); + } + private void basicCancel(Channel channel, String consumerTag) throws IOException { channel.basicCancel(consumerTag); } @@ -309,13 +351,13 @@ public String queueExclusiveDeclareAndBind(String prefix, String exchange) throw return queue; } - private void shutdownSharedExecutor(int closeTimeout) { - sharedExecutor.shutdown(); + private void shutdownExecutor(ExecutorService executor, int closeTimeout, String name) { + executor.shutdown(); try { - if (!sharedExecutor.awaitTermination(closeTimeout, TimeUnit.MILLISECONDS)) { - LOGGER.error("Executor is not terminated during {} millis", closeTimeout); - List runnables = sharedExecutor.shutdownNow(); - LOGGER.error("{} task(s) was(were) not finished", runnables.size()); + if (!executor.awaitTermination(closeTimeout, TimeUnit.MILLISECONDS)) { + LOGGER.error("Executor {} is not terminated during {} millis", name, closeTimeout); + List runnables = executor.shutdownNow(); + LOGGER.error("{} task(s) was(were) not finished in executor {}", runnables.size(), name); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -325,7 +367,7 @@ private void shutdownSharedExecutor(int closeTimeout) { private ChannelHolder getChannelFor(PinId pinId) { return channelsByPin.computeIfAbsent(pinId, ignore -> { LOGGER.trace("Creating channel holder for {}", pinId); - return new ChannelHolder(this::createChannel, this::waitForConnectionRecovery); + return new ChannelHolder(this::createChannel, this::waitForConnectionRecovery, configuration.getPrefetchCount()); }); } @@ -384,7 +426,7 @@ private boolean isConnectionRecovery(ShutdownNotifier notifier) { * deliveries must be acknowledged on the same channel they were received on. * @throws IOException */ - private void basicAck(Channel channel, long deliveryTag) throws IOException { + private static void basicAck(Channel channel, long deliveryTag) throws IOException { channel.basicAck(deliveryTag, false); } @@ -460,20 +502,37 @@ private static class ChannelHolder { private final Lock lock = new ReentrantLock(); private final Supplier supplier; private final BiConsumer reconnectionChecker; + private final int maxCount; + @GuardedBy("lock") + private int pending; + @GuardedBy("lock") + private Future check; + @GuardedBy("lock") private Channel channel; public ChannelHolder( Supplier supplier, - BiConsumer reconnectionChecker + BiConsumer reconnectionChecker, + int maxCount ) { this.supplier = Objects.requireNonNull(supplier, "'Supplier' parameter"); this.reconnectionChecker = Objects.requireNonNull(reconnectionChecker, "'Reconnection checker' parameter"); + this.maxCount = maxCount; } public void withLock(ChannelConsumer consumer) throws IOException { withLock(true, consumer); } + public void withLock(Runnable action) { + lock.lock(); + try { + action.run(); + } finally { + lock.unlock(); + } + } + public void withLock(boolean waitForRecovery, ChannelConsumer consumer) throws IOException { lock.lock(); try { @@ -491,11 +550,62 @@ public T mapWithLock(ChannelMapper mapper) throws IOException { lock.unlock(); } } + + /** + * Decreases the number of unacked messages. + * If the number of unacked messages is less than {@link #maxCount} + * the onWaterMarkDecreased action will be called. + * The future created in {@link #acquireAndSubmitCheck(Supplier)} method will be canceled + * @param onWaterMarkDecreased + * the action that will be executed when the number of unacked messages is less than {@link #maxCount} and there is a future to cancel + */ + public void release(Runnable onWaterMarkDecreased) { + lock.lock(); + try { + pending--; + if (pending < maxCount && check != null) { + check.cancel(true); + check = null; + onWaterMarkDecreased.run(); + } + } finally { + lock.unlock(); + } + } + + /** + * Increases the number of unacked messages. + * If the number of unacked messages is higher than or equal to {@link #maxCount} + * the futureSupplier will be invoked to create a task + * that either will be executed or canceled when number of unacked message will be less that {@link #maxCount} + * @param futureSupplier + * creates a future to track the task that should be executed until the number of unacked message is not less than {@link #maxCount} + */ + public void acquireAndSubmitCheck(Supplier> futureSupplier) { + lock.lock(); + try { + pending++; + if (reachedPendingLimit() && check == null) { + check = futureSupplier.get(); + } + } finally { + lock.unlock(); + } + } + + public boolean reachedPendingLimit() { + lock.lock(); + try { + return pending >= maxCount; + } finally { + lock.unlock(); + } + } + private Channel getChannel() { return getChannel(true); } - private Channel getChannel(boolean waitForRecovery) { if (channel == null) { channel = supplier.get(); diff --git a/src/main/java/com/exactpro/th2/common/schema/strategy/route/RoutingStrategy.java b/src/main/java/com/exactpro/th2/common/schema/strategy/route/RoutingStrategy.java index ca1d34e3b..d5f7b7212 100644 --- a/src/main/java/com/exactpro/th2/common/schema/strategy/route/RoutingStrategy.java +++ b/src/main/java/com/exactpro/th2/common/schema/strategy/route/RoutingStrategy.java @@ -17,7 +17,7 @@ import com.google.protobuf.Message; - +@Deprecated(since = "3.37", forRemoval = true) public interface RoutingStrategy { Class getConfigurationClass(); diff --git a/src/main/java/com/exactpro/th2/common/schema/strategy/route/RoutingStrategyFactory.java b/src/main/java/com/exactpro/th2/common/schema/strategy/route/RoutingStrategyFactory.java index 225c3c155..6b2c47dbe 100644 --- a/src/main/java/com/exactpro/th2/common/schema/strategy/route/RoutingStrategyFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/strategy/route/RoutingStrategyFactory.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Deprecated(since = "3.37", forRemoval = true) public class RoutingStrategyFactory { private static final Logger LOGGER = LoggerFactory.getLogger(RoutingStrategyFactory.class); diff --git a/src/main/java/com/exactpro/th2/common/schema/strategy/route/StrategyName.java b/src/main/java/com/exactpro/th2/common/schema/strategy/route/StrategyName.java index e6f493de1..7e89bf3c5 100644 --- a/src/main/java/com/exactpro/th2/common/schema/strategy/route/StrategyName.java +++ b/src/main/java/com/exactpro/th2/common/schema/strategy/route/StrategyName.java @@ -21,6 +21,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +@Deprecated(since = "3.37", forRemoval = true) @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) public @interface StrategyName { diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/configuration/ConfigurationManager.kt b/src/main/kotlin/com/exactpro/th2/common/schema/configuration/ConfigurationManager.kt index 94becb407..cebcad3db 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/configuration/ConfigurationManager.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/configuration/ConfigurationManager.kt @@ -40,7 +40,7 @@ class ConfigurationManager(private val configurationPath: Map, Path>) { } val sourceContent = String(Files.readAllBytes(configPath)) - LOGGER.info("Configuration path {} source content {}", configPath, sourceContent) + LOGGER.info { "Configuration path $configClass source content $sourceContent" } val content: String = stringSubstitutor.replace(sourceContent) return objectMapper.readerFor(configClass).readValue(content) } catch (e: IOException) { diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/FieldValueChecker.kt b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/FieldValueChecker.kt new file mode 100644 index 000000000..ce49634eb --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/FieldValueChecker.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JvmName("FieldValueChecker") +package com.exactpro.th2.common.schema.filter.strategy.impl + +import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration +import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation +import org.apache.commons.io.FilenameUtils + +fun FieldFilterConfiguration.checkFieldValue(value: String?) = + when (operation) { + FieldFilterOperation.EQUAL -> value == expectedValue + FieldFilterOperation.NOT_EQUAL -> value != expectedValue + FieldFilterOperation.EMPTY -> value.isNullOrEmpty() + FieldFilterOperation.NOT_EMPTY -> !value.isNullOrEmpty() + FieldFilterOperation.WILDCARD -> FilenameUtils.wildcardMatch(value, expectedValue) + } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index d705f6740..0a891a32e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -16,6 +16,7 @@ package com.exactpro.th2.common.schema.grpc.configuration import com.exactpro.th2.common.schema.configuration.Configuration +import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.strategy.route.RoutingStrategy import com.exactpro.th2.service.RetryPolicy import com.fasterxml.jackson.annotation.JsonProperty @@ -27,9 +28,14 @@ data class GrpcConfiguration( ) : Configuration() data class GrpcServiceConfiguration( - @JsonProperty(required = true) var strategy: RoutingStrategy<*>, + @Deprecated("For removal since v3.37") @JsonProperty(required = true) var strategy: RoutingStrategy<*>, @JsonProperty(required = true, value = "service-class") var serviceClass: Class<*>, - @JsonProperty(required = true) var endpoints: Map = emptyMap() + @JsonProperty(required = true) var endpoints: Map = emptyMap(), + @JsonProperty var filters: List = emptyList() +) : Configuration() + +data class Filter( + @JsonProperty(required = true) var properties: List, ) : Configuration() data class GrpcEndpointConfiguration( diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt new file mode 100644 index 000000000..4fc6eceff --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message + +interface ConfirmationMessageListener { + + @Throws(Exception::class) + fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) + + fun onClose() {} + + companion object { + @JvmStatic + fun wrap(listener: MessageListener): ConfirmationMessageListener = DelegateListener(listener) + + /** + * @return `true` if the listener uses manual acknowledgment + */ + @JvmStatic + fun isManual(listener: ConfirmationMessageListener<*>): Boolean = listener is ManualConfirmationListener<*> + } +} + +/** + * The interface marker that indicates that acknowledge will be manually invoked by the listener itself + */ +interface ManualConfirmationListener : ConfirmationMessageListener { + /** + * The listener must invoke the [confirmation] callback once it has processed the [message] + * @see ConfirmationMessageListener.handle + */ + override fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) +} + +private class DelegateListener( + private val delegate: MessageListener, +) : ConfirmationMessageListener { + + override fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { + delegate.handle(consumerTag, message) + } + + override fun onClose() { + delegate.onClose() + } + + override fun toString(): String = "Delegate($delegate)" +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt new file mode 100644 index 000000000..81d1d01bc --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message + +import com.rabbitmq.client.Delivery +import java.io.IOException + +fun interface ManualAckDeliveryCallback { + /** + * Called when a delivery from queue is received + * @param consumerTag the _consumer_ tag associated with the consumer + * @param delivery the delivered message + * @param confirmProcessed the action that should be invoked when the message can be considered as processed + */ + @Throws(IOException::class) + fun handle(consumerTag: String, delivery: Delivery, confirmProcessed: Confirmation) + + fun interface Confirmation { + @Throws(IOException::class) + fun confirm() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt new file mode 100644 index 000000000..7603a39e7 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl + +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback +import mu.KotlinLogging +import java.util.concurrent.atomic.AtomicBoolean + +class OnlyOnceConfirmation private constructor( + private val id: String, + private val delegate: ManualAckDeliveryCallback.Confirmation +) : ManualAckDeliveryCallback.Confirmation { + private val called = AtomicBoolean() + + override fun confirm() { + if (called.compareAndSet(false, true)) { + delegate.confirm() + } else { + LOGGER.warn { "Confirmation '$id' invoked more that one time" } + } + } + + companion object { + private val LOGGER = KotlinLogging.logger { } + + @JvmStatic + fun wrap(id: String, confirmation: ManualAckDeliveryCallback.Confirmation): ManualAckDeliveryCallback.Confirmation = + OnlyOnceConfirmation(id, confirmation) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index f2a98ddcd..76c203e43 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -40,18 +40,16 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { } override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { - return groupBatchRouter.subscribe( - MessageListener { consumerTag: String, message: MessageGroupBatch -> - callback.handler(consumerTag, buildFromGroupBatch(message)) + return groupBatchRouter.subscribe({ consumerTag: String, message: MessageGroupBatch -> + callback.handle(consumerTag, buildFromGroupBatch(message)) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) } override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { - return groupBatchRouter.subscribeAll( - MessageListener { consumerTag: String, message: MessageGroupBatch -> - callback.handler(consumerTag, buildFromGroupBatch(message)) + return groupBatchRouter.subscribeAll({ consumerTag: String, message: MessageGroupBatch -> + callback.handle(consumerTag, buildFromGroupBatch(message)) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index c233298c4..6f3c66730 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -17,9 +17,16 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.exception.RouterException import com.exactpro.th2.common.schema.filter.strategy.FilterStrategy -import com.exactpro.th2.common.schema.message.* +import com.exactpro.th2.common.schema.message.ConfirmationMessageListener +import com.exactpro.th2.common.schema.message.MessageListener +import com.exactpro.th2.common.schema.message.MessageRouter +import com.exactpro.th2.common.schema.message.MessageRouterContext +import com.exactpro.th2.common.schema.message.MessageSender +import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.QueueAttribute.PUBLISH import com.exactpro.th2.common.schema.message.QueueAttribute.SUBSCRIBE +import com.exactpro.th2.common.schema.message.SubscriberMonitor +import com.exactpro.th2.common.schema.message.appendAttributes import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter @@ -55,7 +62,7 @@ abstract class AbstractRabbitRouter : MessageRouter { private val subscribers = ConcurrentHashMap>() private val senders = ConcurrentHashMap>() - private val filterStrategy = AtomicReference>(getDefaultFilterStrategy()) + private val filterStrategy = AtomicReference(getDefaultFilterStrategy()) protected open fun getDefaultFilterStrategy(): FilterStrategy { return FilterStrategy.DEFAULT_FILTER_STRATEGY @@ -72,7 +79,7 @@ abstract class AbstractRabbitRouter : MessageRouter { override fun send(message: T, vararg attributes: String) { val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSendAttributes() } send(message, pintAttributes) { - check(size == 1 || (size > 0 && oneOrNoneWithData())) { + check(size == 1 || (isNotEmpty() && oneOrNoneWithData())) { "Found incorrect number of pins ${map(PinInfo::pinName)} to the send operation by attributes $pintAttributes and filters, expected 1, actual $size" } } @@ -88,6 +95,14 @@ abstract class AbstractRabbitRouter : MessageRouter { } override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor { + return subscribeWithManualAck(ConfirmationMessageListener.wrap(callback), *attributes) + } + + override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor { + return subscribeAllWithManualAck(ConfirmationMessageListener.wrap(callback), *attributes) + } + + override fun subscribeWithManualAck(callback: ConfirmationMessageListener, vararg attributes: String): SubscriberMonitor { val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } return subscribe(pintAttributes, callback) { check(size == 1) { @@ -96,7 +111,7 @@ abstract class AbstractRabbitRouter : MessageRouter { } } - override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { + override fun subscribeAllWithManualAck(callback: ConfirmationMessageListener, vararg attributes: String): SubscriberMonitor { val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } return subscribe(pintAttributes, callback) { check(isNotEmpty()) { @@ -115,7 +130,7 @@ abstract class AbstractRabbitRouter : MessageRouter { } subscribers.clear() - checkOrThrow("Can not close message router", exceptions) + checkOrThrow(exceptions) { "Can not close message router" } LOGGER.info("Message router has been successfully closed") } @@ -171,12 +186,12 @@ abstract class AbstractRabbitRouter : MessageRouter { exceptions[pinName] = e } } - checkOrThrow("Can't send to pin(s): ${exceptions.keys}", exceptions.values) + checkOrThrow(exceptions.values) { "Can't send to pin(s): ${exceptions.keys}" } } private fun subscribe( pintAttributes: Set, - messageListener: MessageListener, + messageListener: ConfirmationMessageListener, check: List.() -> Unit ): SubscriberMonitor { val packages: List = configuration.queues.asSequence() @@ -202,7 +217,7 @@ abstract class AbstractRabbitRouter : MessageRouter { } } - checkOrThrow("Can't subscribe to pin(s): ${exceptions.keys}", exceptions.values) + checkOrThrow(exceptions.values) { "Can't subscribe to pin(s): ${exceptions.keys}" } return when (monitors.size) { 1 -> monitors[0] @@ -212,9 +227,9 @@ abstract class AbstractRabbitRouter : MessageRouter { } } - private fun checkOrThrow(message: String, exceptions: Collection) { + private inline fun checkOrThrow(exceptions: Collection, message: () -> String) { if (exceptions.isNotEmpty()) { - throw RouterException(message).apply { + throw RouterException(message()).apply { exceptions.forEach(this::addSuppressed) } } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/configuration/RabbitMQConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/configuration/RabbitMQConfiguration.kt index d7d60395c..a8d2813be 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/configuration/RabbitMQConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/configuration/RabbitMQConfiguration.kt @@ -17,6 +17,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration import com.exactpro.th2.common.schema.configuration.Configuration import com.fasterxml.jackson.annotation.JsonProperty +import java.time.Duration data class RabbitMQConfiguration( @JsonProperty(required = true) var host: String, @@ -36,5 +37,12 @@ data class ConnectionManagerConfiguration( var minConnectionRecoveryTimeout: Int = 10000, var maxConnectionRecoveryTimeout: Int = 60000, val prefetchCount: Int = 10, - val messageRecursionLimit: Int = 100 -) : Configuration() \ No newline at end of file + val messageRecursionLimit: Int = 100, + val workingThreads: Int = 1, + val confirmationTimeout: Duration = Duration.ofMinutes(5) +) : Configuration() { + init { + check(workingThreads > 0) { "expected 'workingThreads' greater than 0 but was $workingThreads" } + check(!confirmationTimeout.run { isNegative || isZero }) { "expected 'confirmationTimeout' greater than 0 but was $confirmationTimeout" } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt index 19c00dd44..a1e39157f 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -27,6 +27,7 @@ import com.exactpro.th2.common.schema.message.FilterFunction import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback import com.exactpro.th2.common.schema.message.impl.rabbitmq.group.RabbitMessageGroupBatchRouter.Companion.MESSAGE_GROUP_TYPE import com.exactpro.th2.common.schema.message.toShortDebugString import com.google.protobuf.CodedInputStream @@ -77,7 +78,12 @@ class RabbitMessageGroupBatchSubscriber( return if (groups.isEmpty()) null else MessageGroupBatch.newBuilder().addAllGroups(groups).build() } - override fun handle(consumeTag: String, delivery: Delivery, value: MessageGroupBatch) { + override fun handle( + consumeTag: String, + delivery: Delivery, + value: MessageGroupBatch, + confirmation: ManualAckDeliveryCallback.Confirmation + ) { incrementTotalMetrics( value, th2Pin, @@ -85,7 +91,7 @@ class RabbitMessageGroupBatchSubscriber( MESSAGE_GROUP_SUBSCRIBE_TOTAL, MESSAGE_GROUP_SEQUENCE_SUBSCRIBE ) - super.handle(consumeTag, delivery, value) + super.handle(consumeTag, delivery, value, confirmation) } private fun parseEncodedBatch(body: ByteArray?): MessageGroupBatch { diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt index f44f141a5..b81b06642 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -17,6 +17,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification import com.exactpro.th2.common.grpc.EventBatch import com.exactpro.th2.common.schema.exception.RouterException +import com.exactpro.th2.common.schema.message.ConfirmationMessageListener import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouterContext import com.exactpro.th2.common.schema.message.NotificationRouter @@ -54,7 +55,7 @@ class NotificationEventBatchRouter : NotificationRouter { override fun subscribe(callback: MessageListener): SubscriberMonitor { try { - subscriber.addListener(callback) + subscriber.addListener(ConfirmationMessageListener.wrap(callback)) subscriber.start() } catch (e: Exception) { val errorMessage = "Listener can't be subscribed via the queue $queue" diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt index 8870b854c..dff106246 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,9 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification import com.exactpro.th2.common.grpc.EventBatch +import com.exactpro.th2.common.schema.message.ConfirmationMessageListener import com.exactpro.th2.common.schema.message.FilterFunction -import com.exactpro.th2.common.schema.message.MessageListener +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.SubscriberMonitor import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget @@ -31,7 +32,7 @@ class NotificationEventBatchSubscriber( private val connectionManager: ConnectionManager, private val queue: String ) : MessageSubscriber { - private val listeners = CopyOnWriteArrayList>() + private val listeners = CopyOnWriteArrayList>() private lateinit var monitor: SubscriberMonitor @Deprecated( @@ -57,30 +58,34 @@ class NotificationEventBatchSubscriber( override fun start() { monitor = connectionManager.basicConsume( queue, - { consumerTag: String, delivery: Delivery -> - for (listener in listeners) { - try { - listener.handler(consumerTag, EventBatch.parseFrom(delivery.body)) - } catch (listenerExc: Exception) { - LOGGER.warn( - "Message listener from class '{}' threw exception", - listener.javaClass, - listenerExc - ) + { consumerTag: String, delivery: Delivery, confirmation: ManualAckDeliveryCallback.Confirmation -> + try { + for (listener in listeners) { + try { + listener.handle(consumerTag, EventBatch.parseFrom(delivery.body), confirmation) + } catch (listenerExc: Exception) { + LOGGER.warn( + "Message listener from class '{}' threw exception", + listener.javaClass, + listenerExc + ) + } } + } finally { + confirmation.confirm() } }, { LOGGER.warn("Consuming cancelled for: '{}'", it) } ) } - override fun addListener(messageListener: MessageListener) { + override fun addListener(messageListener: ConfirmationMessageListener) { listeners.add(messageListener) } override fun close() { monitor.unsubscribe() - listeners.forEach(MessageListener::onClose) + listeners.forEach(ConfirmationMessageListener::onClose) listeners.clear() } diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index b98dd6a1a..34407aef6 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -1,5 +1,5 @@ ################################################################################ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. ################################################################################ -log4j.rootLogger=INFO, CON +log4j.rootLogger=INFO, CON, PROMETHEUS log4j.appender.CON=org.apache.log4j.ConsoleAppender log4j.appender.CON.layout=org.apache.log4j.PatternLayout -log4j.appender.CON.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n \ No newline at end of file +log4j.appender.CON.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n + +log4j.appender.PROMETHEUS=io.prometheus.client.log4j.InstrumentedAppender \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt b/src/test/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt new file mode 100644 index 000000000..8603d1e78 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.annotations + +import org.junit.jupiter.api.Tag + +@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.RUNTIME) +@Tag("integration-test") +annotation class IntegrationTest diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index 24ebd1a41..91831bf87 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -34,6 +34,7 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.Rabbit import com.exactpro.th2.common.schema.strategy.route.impl.RobinRoutingStrategy import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.KotlinModule import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -124,6 +125,7 @@ class TestJsonConfiguration { companion object { @JvmStatic private val OBJECT_MAPPER: ObjectMapper = ObjectMapper() + .registerModule(JavaTimeModule()) @JvmStatic private val CONF_DIR = Path.of("test_json_configurations") @@ -136,7 +138,8 @@ class TestJsonConfiguration { init(GrpcRawRobinStrategy(listOf("endpoint"))) }, GrpcConfiguration::class.java, - mapOf("endpoint" to GrpcEndpointConfiguration("host", 12345, listOf("test_attr"))) + mapOf("endpoint" to GrpcEndpointConfiguration("host", 12345, listOf("test_attr"))), + emptyList() ) ), GrpcServerConfiguration("host123", 1234, 58) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt new file mode 100644 index 000000000..0f6cd08e3 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify + +class TestConfirmationMessageListenerWrapper { + @Test + fun `calls confirmation when requested`() { + val listener = ConfirmationMessageListener.wrap { _, _ -> } + + mock {}.also { + listener.handle("", 2, it) + verify(it, never()).confirm() + } + } + + @Test + fun `calls confirmation when requested and method throw an exception`() { + val listener = ConfirmationMessageListener.wrap { _, _ -> error("test") } + + mock {}.also { + assertThrows(IllegalStateException::class.java) { listener.handle("", 2, it) }.apply { + assertEquals("test", message) + } + verify(it, never()).confirm() + } + } + + +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/TestConnectionManager.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/TestConnectionManager.kt new file mode 100644 index 000000000..fe805a0af --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/TestConnectionManager.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.message.impl.rabbitmq.connection + +import com.exactpro.th2.common.annotations.IntegrationTest +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration +import com.rabbitmq.client.BuiltinExchangeType +import mu.KotlinLogging +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName +import java.time.Duration +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +private val LOGGER = KotlinLogging.logger { } + + +@IntegrationTest +class TestConnectionManager { + + @Test + fun `connection manager reports unacked messages when confirmation timeout elapsed`() { + val routingKey = "routingKey" + val queueName = "queue" + val exchange = "test-exchange" + val prefetchCount = 10 + RabbitMQContainer(DockerImageName.parse("rabbitmq:3.8-management-alpine")) + .withExchange(exchange, BuiltinExchangeType.FANOUT.type, false, false, true, emptyMap()) + .withQueue(queueName) + .withBinding(exchange, queueName, emptyMap(), routingKey, "queue") + .use { + it.start() + LOGGER.info { "Started with port ${it.amqpPort}" } + val queue = ArrayBlockingQueue(prefetchCount) + val countDown = CountDownLatch(prefetchCount) + val confirmationTimeout = Duration.ofSeconds(1) + ConnectionManager( + RabbitMQConfiguration( + host = it.host, + vHost = "", + port = it.amqpPort, + username = it.adminUsername, + password = it.adminPassword, + ), + ConnectionManagerConfiguration( + subscriberName = "test", + prefetchCount = prefetchCount, + confirmationTimeout = confirmationTimeout, + ), + ) { + LOGGER.error { "Fatal connection problem" } + }.use { manager -> + manager.basicConsume(queueName, { _, delivery, ack -> + LOGGER.info { "Received ${delivery.body.toString(Charsets.UTF_8)} from ${delivery.envelope.routingKey}" } + queue += ack + countDown.countDown() + }) { + LOGGER.info { "Canceled $it" } + } + + repeat(prefetchCount + 1) { index -> + manager.basicPublish(exchange, routingKey, null, "Hello $index".toByteArray(Charsets.UTF_8)) + } + + Assertions.assertTrue(countDown.await(1L, TimeUnit.SECONDS)) { "Not all messages were received: ${countDown.count}" } + + Assertions.assertTrue(manager.isAlive) { "Manager should still be alive" } + Assertions.assertTrue(manager.isReady) { "Manager should be ready until the confirmation timeout expires" } + + Thread.sleep(confirmationTimeout.toMillis() + 100/*just in case*/) // wait for confirmation timeout + + Assertions.assertTrue(manager.isAlive) { "Manager should still be alive" } + Assertions.assertFalse(manager.isReady) { "Manager should not be ready" } + + queue.poll().confirm() + + Assertions.assertTrue(manager.isAlive) { "Manager should still be alive" } + Assertions.assertTrue(manager.isReady) { "Manager should be ready" } + + val receivedData = generateSequence { queue.poll(10L, TimeUnit.MILLISECONDS) } + .onEach(ManualAckDeliveryCallback.Confirmation::confirm) + .count() + Assertions.assertEquals(prefetchCount, receivedData) { "Unexpected number of messages received" } + } + } + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageConverterLambdaDelegate.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageConverterLambdaDelegate.kt index fcae0fa64..91ad69184 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageConverterLambdaDelegate.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/TestMessageConverterLambdaDelegate.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,8 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.custom -import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll From 78ae0557e978aa4212807658838189f79a68fcdd Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Wed, 25 May 2022 15:37:59 +0400 Subject: [PATCH 026/154] [TH2-3685] Updated cradle version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7aff2d717..a7d0c8b20 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-2269130595-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-2356983195-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 21243ea815c0471fdad08f77d53844061c99b664 Mon Sep 17 00:00:00 2001 From: Rekish Date: Tue, 31 May 2022 15:30:36 +0400 Subject: [PATCH 027/154] [TH2-3436] added aliases for fields in json config --- .../message/configuration/MessageRouterConfiguration.kt | 4 ++-- .../resources/test_json_configurations/message_router.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt index 1b5e2a39d..fabed6f93 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt @@ -82,8 +82,8 @@ data class FieldFilterConfigurationOld( ) : Configuration() data class FieldFilterConfiguration( - @JsonProperty(value = "fieldName", required = true) var fieldName: String, - @JsonProperty("expectedValue") @JsonAlias("value") var expectedValue: String?, + @JsonProperty(value = "fieldName", required = true) @JsonAlias("fieldName", "field-name") var fieldName: String, + @JsonProperty("expectedValue") @JsonAlias("value", "expected-value") var expectedValue: String?, @JsonProperty(required = true) var operation: FieldFilterOperation ) : Configuration() diff --git a/src/test/resources/test_json_configurations/message_router.json b/src/test/resources/test_json_configurations/message_router.json index dc8a69397..bb5e0cf2f 100644 --- a/src/test/resources/test_json_configurations/message_router.json +++ b/src/test/resources/test_json_configurations/message_router.json @@ -23,8 +23,8 @@ { "metadata": [ { - "fieldName": "session_alias", - "value": "test_session_alias", + "field-name": "session_alias", + "expected-value": "test_session_alias", "operation": "EQUAL" } ], From cd6146ea557c373ebe2a39cc81056c86bc1e6c37 Mon Sep 17 00:00:00 2001 From: nChurgulia Date: Wed, 8 Jun 2022 21:19:49 +0400 Subject: [PATCH 028/154] cradle version updated --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e3c283fd9..f6f7e9a38 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '3.2.0-ver_3_1_dev-2300230098-SNAPSHOT' + cradleVersion = '3.2.0-TH2-3649-3.2v-2463202212-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 78beaac9369b7c2e9d4bb692fcdc8e11aae78e30 Mon Sep 17 00:00:00 2001 From: nChurgulia Date: Fri, 10 Jun 2022 14:08:34 +0400 Subject: [PATCH 029/154] [TH2-3649] cradle version updated --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f6f7e9a38..c522deef1 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '3.2.0-TH2-3649-3.2v-2463202212-SNAPSHOT' + cradleVersion = '3.2.1' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From c91b82859e21327308d2e3babd480bab70fe2cd9 Mon Sep 17 00:00:00 2001 From: nChurgulia <45267359+nChurgulia@users.noreply.github.com> Date: Fri, 10 Jun 2022 15:44:21 +0400 Subject: [PATCH 030/154] [TH2-3649-4.0v] cradle version updated (#215) * cradle version updated * [TH2-3649] cradle version updated * [TH2-3649] cradle version updated --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a7d0c8b20..ac4e793ee 100644 --- a/build.gradle +++ b/build.gradle @@ -30,11 +30,12 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-2356983195-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-2474671990-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } + repositories { mavenCentral() maven { From 96921efba6541364fb3d79d7fcbbc73a565a1b55 Mon Sep 17 00:00:00 2001 From: georgiano Date: Tue, 14 Jun 2022 19:18:43 +0400 Subject: [PATCH 031/154] [TH2-3776] updated cradle library with reworked groups handling --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ac4e793ee..43f8c659b 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-2474671990-SNAPSHOT' + cradleVersion = '4.0.0-TH2-2150-2496249472-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 1499501da2661d1cc0ab9318c90514effd06eae5 Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Wed, 15 Jun 2022 19:15:18 +0400 Subject: [PATCH 032/154] [TH2-3789] Updated cradle version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c522deef1..b6e119dfd 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '3.2.1' + cradleVersion = '3.2.1-TH2-3789-2503185221-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 542352b6c78c642ad5c031a5de12d60096e37ab5 Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Thu, 16 Jun 2022 16:05:19 +0400 Subject: [PATCH 033/154] [TH2-3789] Updated cradle version (cradle fix) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b6e119dfd..590594b76 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '3.2.1-TH2-3789-2503185221-SNAPSHOT' + cradleVersion = '3.2.2-TH2-3789-2508815656-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 992335b627f9b786ec05f2e0506c8905c5033d5a Mon Sep 17 00:00:00 2001 From: georgiano Date: Mon, 20 Jun 2022 15:10:03 +0400 Subject: [PATCH 034/154] [TH2-3822] replaced cradle library with 5.0.0 snapshot version --- build.gradle | 2 +- gradle.properties | 2 +- .../th2/common/schema/factory/AbstractCommonFactory.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 43f8c659b..499e5606a 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-2496249472-SNAPSHOT' + cradleVersion = '5.0.0-TH2-3822-2527812177-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } diff --git a/gradle.properties b/gradle.properties index a72889179..6f53c9cb2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=4.0.0 +release_version=5.0.0 description = 'th2 common library (Java)' diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 554cbb5a1..b6cb761cf 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -524,7 +524,7 @@ public CradleManager getCradleManager() { DEFAULT_CONSISTENCY_LEVEL, DEFAULT_CONSISTENCY_LEVEL ); - cassandraStorageSettings.setCradleInfoKeyspace(confidentialConfiguration.getKeyspace()); + cassandraStorageSettings.setKeyspace(confidentialConfiguration.getKeyspace()); if (nonConfidentialConfiguration.getPageSize() > 0) { cassandraStorageSettings.setResultPageSize(nonConfidentialConfiguration.getPageSize()); } From 9255f67c0d9e7707c026db1e244dd3c01115a86d Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Mon, 20 Jun 2022 18:40:36 +0400 Subject: [PATCH 035/154] [TH2-3789] Updated cradle version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 590594b76..4f75c7d99 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '3.2.2-TH2-3789-2508815656-SNAPSHOT' + cradleVersion = '3.2.2-TH2-3789-2514887874-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 67df98cc4c8eb44c73c2d70c5063e9f2165ec806 Mon Sep 17 00:00:00 2001 From: georgiano Date: Tue, 21 Jun 2022 13:06:13 +0400 Subject: [PATCH 036/154] [TH2-3822] updated cradle library --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 499e5606a..eed4ae6f0 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '5.0.0-TH2-3822-2527812177-SNAPSHOT' + cradleVersion = '5.0.0-dev-version-5-2533886155-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From a9a8877ee3e3d6c3827ca819fdf6bff5abb2f97c Mon Sep 17 00:00:00 2001 From: georgiano Date: Wed, 22 Jun 2022 13:58:16 +0400 Subject: [PATCH 037/154] [TH2-3822] updated cradle library --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index eed4ae6f0..fa340d126 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '5.0.0-dev-version-5-2533886155-SNAPSHOT' + cradleVersion = '5.0.0-dev-version-5-2541270526-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 0ad57e7d6be67058569300e3d6669d24e8e85863 Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Thu, 23 Jun 2022 16:27:49 +0400 Subject: [PATCH 038/154] [TH2-3789] Updated cradle version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4f75c7d99..28b5b4655 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '3.2.2-TH2-3789-2514887874-SNAPSHOT' + cradleVersion = '3.2.2-TH2-3789-2548930617-SNAPSHOT' junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From 90cf4a391316a4b1e5f22e3995a406e42c7a1d7b Mon Sep 17 00:00:00 2001 From: georgiano Date: Mon, 27 Jun 2022 12:04:29 +0400 Subject: [PATCH 039/154] [TH2-3822] updated cradle --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index fa340d126..40e8b1bac 100644 --- a/build.gradle +++ b/build.gradle @@ -30,8 +30,8 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '5.0.0-dev-version-5-2541270526-SNAPSHOT' - junitVersion = '5.7.2' + cradleVersion = '5.0.0-dev-version-5-2567311063-SNAPSHOT' + junitVersion = '5.7.2' sharedDir = file("${project.rootDir}/shared") } From c11aeec08c06417f3865983740446ac7931199a7 Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Mon, 27 Jun 2022 12:11:41 +0400 Subject: [PATCH 040/154] [TH2-3789] updated README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4bf3cd0f2..136287439 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (3.39.2) +# th2 common library (Java) (3.39.3) ## Usage @@ -298,6 +298,10 @@ dependencies { ## Release notes +### 3.39.3 + ++ Started using cradle 3.2, version with grouped messages + ### 3.39.2 + Fixed: From 8a19ade30aca38a676e418d06026b5ff67a2784b Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Mon, 27 Jun 2022 13:09:23 +0400 Subject: [PATCH 041/154] [TH2-3789] increased minor version instead of patch --- README.md | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 136287439..f728f4943 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ dependencies { ## Release notes -### 3.39.3 +### 3.40.0 + Started using cradle 3.2, version with grouped messages diff --git a/gradle.properties b/gradle.properties index afc218890..16c6ebf27 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.39.3 +release_version=3.40.0 description = 'th2 common library (Java)' From 4b22e22e4278d2e78582f90ad4e34d8d13267a91 Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Mon, 27 Jun 2022 13:10:25 +0400 Subject: [PATCH 042/154] [TH2-3789] fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f728f4943..75616d453 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (3.39.3) +# th2 common library (Java) (3.40.0) ## Usage From 871ad2aef0848ec438da5527221ac6ff94dfb64f Mon Sep 17 00:00:00 2001 From: Rekish Date: Mon, 27 Jun 2022 14:08:43 +0400 Subject: [PATCH 043/154] deleted log4j.properties --- src/main/resources/log4j.properties | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/main/resources/log4j.properties diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties deleted file mode 100644 index e69de29bb..000000000 From 4450e663acf59b6fa354ad511ef4cdc3be790704 Mon Sep 17 00:00:00 2001 From: nChurgulia Date: Mon, 27 Jun 2022 17:26:26 +0400 Subject: [PATCH 044/154] [TH2-3833] cradle version updated --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 32e5878b6..4f1afa761 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ sourceCompatibility = 11 targetCompatibility = 11 ext { - cradleVersion = '4.0.0-TH2-2150-2496249472-SNAPSHOT' + cradleVersion = '4.0.0-dev-version-4-2548754188-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } From d2f58fb6ebf05a23e1962c1eca87b792a6104612 Mon Sep 17 00:00:00 2001 From: Rekish Date: Thu, 7 Jul 2022 15:00:12 +0400 Subject: [PATCH 045/154] [TH2-2909] added dependencies for log4j2 --- build.gradle | 4 ++++ .../com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt | 2 ++ 2 files changed, 6 insertions(+) diff --git a/build.gradle b/build.gradle index 4f1afa761..b7b66ffd0 100644 --- a/build.gradle +++ b/build.gradle @@ -202,6 +202,10 @@ dependencies { implementation 'io.github.microutils:kotlin-logging:2.1.21' + implementation 'org.apache.logging.log4j:log4j-slf4j-impl' + implementation 'org.apache.logging.log4j:log4j-1.2-api' + implementation 'org.apache.logging.log4j:log4j-api' + implementation 'io.prometheus:simpleclient' implementation 'io.prometheus:simpleclient_hotspot' implementation 'io.prometheus:simpleclient_httpserver' diff --git a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt index d6e081c10..3e5159055 100644 --- a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt +++ b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt @@ -59,6 +59,7 @@ class Log4jConfigUtils { private fun configureFirstLog4j(path: Path) { try { + LOGGER.info("Trying to apply logger config from {}. Expecting log4j syntax", path) PropertyConfigurator.configure(path.toUri().toURL()) LOGGER.info("Logger configuration from {} file is applied", path) } catch (e: MalformedURLException) { @@ -67,6 +68,7 @@ class Log4jConfigUtils { } private fun configureSecondLog4j(path: Path) { + LOGGER.info("Trying to apply logger config from {}. Expecting log4j2 syntax", path) val context = LogManager.getContext(false) as LoggerContext context.configLocation = path.toUri() LOGGER.info("Logger configuration from {} file is applied", path) From 813df7e2509b72c1898b468023a441576e3939f4 Mon Sep 17 00:00:00 2001 From: Rekish Date: Fri, 8 Jul 2022 14:57:01 +0400 Subject: [PATCH 046/154] [TH2-3877] grpc-common version bump --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b7b66ffd0..649fafa3e 100644 --- a/build.gradle +++ b/build.gradle @@ -168,7 +168,7 @@ tasks.register('integrationTest', Test) { dependencies { api platform('com.exactpro.th2:bom:3.2.0') api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'com.exactpro.th2:grpc-common:4.0.0-th2-2150-books-pages-2338290872-SNAPSHOT' + api 'com.exactpro.th2:grpc-common:4.0.0-th2-2150-books-pages-2635598705-SNAPSHOT' implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.2.2' From a991490a35205166ead8a8f21e3c8a76c60955a1 Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Fri, 5 Aug 2022 15:07:40 +0400 Subject: [PATCH 047/154] [TH2-4068] Add ability to declare keep alive timeout option for grpc client configuration. --- .../th2/common/schema/grpc/router/impl/DefaultStubStorage.java | 2 ++ .../th2/common/schema/grpc/configuration/GrpcConfiguration.kt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java index e0d6993fd..9dbf96220 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.annotation.concurrent.ThreadSafe; @@ -114,6 +115,7 @@ public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory return stubFactory.newStub( ManagedChannelBuilder.forAddress(endpoint.getHost(), endpoint.getPort()) + .keepAliveTime(service.serviceConfig.getKeepAliveInterval(), TimeUnit.SECONDS) .usePlaintext() .intercept(new GrpcInterceptor(service.pinName, methodInvokeCounter, requestBytesCounter, responseBytesCounter)) .build(), diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index 0a891a32e..ecfb04165 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -31,7 +31,8 @@ data class GrpcServiceConfiguration( @Deprecated("For removal since v3.37") @JsonProperty(required = true) var strategy: RoutingStrategy<*>, @JsonProperty(required = true, value = "service-class") var serviceClass: Class<*>, @JsonProperty(required = true) var endpoints: Map = emptyMap(), - @JsonProperty var filters: List = emptyList() + @JsonProperty var filters: List = emptyList(), + @JsonProperty(required = false) var keepAliveInterval: Long = 300L ) : Configuration() data class Filter( From d8108eb8d57a3c00b8aa41c5f6997968443a9277 Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Fri, 5 Aug 2022 15:50:24 +0400 Subject: [PATCH 048/154] Bump version and readme update. --- README.md | 3 +++ gradle.properties | 2 +- .../schema/grpc/router/impl/DefaultGrpcRouter.java | 6 +++++- .../schema/grpc/router/impl/DefaultStubStorage.java | 10 +++++++--- .../schema/grpc/configuration/GrpcConfiguration.kt | 8 ++++++-- 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 81ac9c2ac..9bc833923 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,9 @@ dependencies { ## Release notes +### 3.40.1 ++ gRPC configuration now contains `client` property where `keepAliveInterval` can be defined for client grpc services. (default: 300 sec) + ### 3.40.0 + gRPC router creates server support [grpc-service-reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md#grpc-server-reflection-tutorial) diff --git a/gradle.properties b/gradle.properties index 970584b98..c63ef9616 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.40.0 +release_version=3.40.1 description = 'th2 common library (Java)' diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java index 0ef44d2ea..ca3779d0d 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java @@ -85,7 +85,11 @@ public T getService(@NotNull Class cls) throws ClassNotFoundException { .newInstance( configuration.getRetryConfiguration(), stubsStorages.computeIfAbsent(cls, key -> - new DefaultStubStorage<>(getServiceConfig(key), GRPC_INVOKE_CALL_TOTAL, GRPC_INVOKE_CALL_REQUEST_BYTES, GRPC_INVOKE_CALL_RESPONSE_BYTES) + new DefaultStubStorage<>(getServiceConfig(key), + GRPC_INVOKE_CALL_TOTAL, + GRPC_INVOKE_CALL_REQUEST_BYTES, + GRPC_INVOKE_CALL_RESPONSE_BYTES, + configuration.getClientConfiguration()) ) ); } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java index 9dbf96220..ce4278558 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java @@ -29,6 +29,7 @@ import com.exactpro.th2.common.grpc.router.GrpcInterceptor; import com.exactpro.th2.common.schema.filter.strategy.impl.FieldValueChecker; +import com.exactpro.th2.common.schema.grpc.configuration.GrpcClientConfiguration; import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration; import io.prometheus.client.Counter; import org.jetbrains.annotations.NotNull; @@ -57,6 +58,7 @@ private static class ServiceHolder { } private final List> services; + private final GrpcClientConfiguration clientConfiguration; private final Counter methodInvokeCounter; private final Counter requestBytesCounter; private final Counter responseBytesCounter; @@ -66,11 +68,13 @@ public DefaultStubStorage( @NotNull List> serviceConfigurations, @NotNull Counter methodInvokeCounter, @NotNull Counter requestBytesCounter, - @NotNull Counter responseBytesCounter - ) { + @NotNull Counter responseBytesCounter, + @NotNull GrpcClientConfiguration clientConfiguration + ) { this.methodInvokeCounter = methodInvokeCounter; this.requestBytesCounter = requestBytesCounter; this.responseBytesCounter = responseBytesCounter; + this.clientConfiguration = clientConfiguration; services = new ArrayList<>(serviceConfigurations.size()); for (final var config: serviceConfigurations) { @@ -115,7 +119,7 @@ public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory return stubFactory.newStub( ManagedChannelBuilder.forAddress(endpoint.getHost(), endpoint.getPort()) - .keepAliveTime(service.serviceConfig.getKeepAliveInterval(), TimeUnit.SECONDS) + .keepAliveTime(clientConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .usePlaintext() .intercept(new GrpcInterceptor(service.pinName, methodInvokeCounter, requestBytesCounter, responseBytesCounter)) .build(), diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index ecfb04165..3e50ea20c 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty data class GrpcConfiguration( @JsonProperty var services: Map = emptyMap(), @JsonProperty(value = "server") var serverConfiguration: GrpcServerConfiguration = GrpcServerConfiguration(), + @JsonProperty(value = "client") var clientConfiguration: GrpcClientConfiguration = GrpcClientConfiguration(), @JsonProperty var retryConfiguration: GrpcRetryConfiguration = GrpcRetryConfiguration() ) : Configuration() @@ -31,14 +32,17 @@ data class GrpcServiceConfiguration( @Deprecated("For removal since v3.37") @JsonProperty(required = true) var strategy: RoutingStrategy<*>, @JsonProperty(required = true, value = "service-class") var serviceClass: Class<*>, @JsonProperty(required = true) var endpoints: Map = emptyMap(), - @JsonProperty var filters: List = emptyList(), - @JsonProperty(required = false) var keepAliveInterval: Long = 300L + @JsonProperty var filters: List = emptyList() ) : Configuration() data class Filter( @JsonProperty(required = true) var properties: List, ) : Configuration() +data class GrpcClientConfiguration( + @JsonProperty var keepAliveInterval: Long = 300L +) + data class GrpcEndpointConfiguration( @JsonProperty(required = true) var host: String, @JsonProperty(required = true) var port: Int = 8080, From 013af3ae399e4b3ae5d89538d1a1097c3e398d1e Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Mon, 8 Aug 2022 14:21:02 +0400 Subject: [PATCH 049/154] [TH2-3789] MessageUtils > add sessionGroup extension property for Message/RawMessage --- .../exactpro/th2/common/message/MessageUtils.kt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt index a3fc8076a..3ebb74611 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt @@ -329,6 +329,22 @@ var RawMessage.Builder.sessionAlias metadataBuilder.idBuilder.connectionIdBuilder.sessionAlias = value } +val Message.sessionGroup + get(): String = metadata.id.connectionId.sessionGroup +var Message.Builder.sessionGroup + get(): String = metadata.id.connectionId.sessionGroup + set(value) { + metadataBuilder.idBuilder.connectionIdBuilder.sessionGroup = value + } + +val RawMessage.sessionGroup + get(): String = metadata.id.connectionId.sessionGroup +var RawMessage.Builder.sessionGroup + get(): String = metadata.id.connectionId.sessionGroup + set(value) { + metadataBuilder.idBuilder.connectionIdBuilder.sessionGroup = value + } + val Message.sequence get(): Long = metadata.id.sequence var Message.Builder.sequence From eb6eaeed10cc846ad3a4ce7c9716ceabaca2bfe2 Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Tue, 9 Aug 2022 13:22:50 +0400 Subject: [PATCH 050/154] Readme updated with grpc configuration description. --- README.md | 39 +++++++++++++++++++ .../grpc/configuration/GrpcConfiguration.kt | 2 +- .../common/schema/TestJsonConfiguration.kt | 4 +- .../test_json_configurations/grpc.json | 3 ++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9bc833923..9fb65d6fc 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,45 @@ Filters format: } ``` +The `CommonFactory` reads a grpc's router configuration from the `grpc.json` file. +* services - grpc services configurations +* server - grpc server configuration +* client - grpc client configuration + * keepAliveInterval - number of seconds before each keep alive message. + +```json +{ + "services": { + "test": { + "endpoints": { + "endpoint": { + "host": "host", + "port": 12345, + "attributes": [ + "test_attr" + ] + } + }, + "service-class": "com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration", + "strategy": { + "endpoints": [ + "endpoint" + ], + "name": "robin" + } + } + }, + "server": { + "host": "host123", + "port": 1234, + "workers": 58 + }, + "client": { + "keepAliveInterval": 400 + } +} +``` + The `CommonFactory` reads a Cradle configuration from the cradle.json file. * dataCenter - the required setting defines the data center in the Cassandra cluster. * host - the required setting defines the Cassandra host. diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index 3e50ea20c..f8bcce298 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -40,7 +40,7 @@ data class Filter( ) : Configuration() data class GrpcClientConfiguration( - @JsonProperty var keepAliveInterval: Long = 300L + @JsonProperty var keepAliveInterval: Long = 60L ) data class GrpcEndpointConfiguration( diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index 16a5857b4..fcadeabef 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -18,6 +18,7 @@ package com.exactpro.th2.common.schema import com.exactpro.th2.common.metrics.PrometheusConfiguration import com.exactpro.th2.common.schema.cradle.CradleConfidentialConfiguration import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration +import com.exactpro.th2.common.schema.grpc.configuration.GrpcClientConfiguration import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration import com.exactpro.th2.common.schema.grpc.configuration.GrpcEndpointConfiguration import com.exactpro.th2.common.schema.grpc.configuration.GrpcRawRobinStrategy @@ -141,7 +142,8 @@ class TestJsonConfiguration { emptyList() ) ), - GrpcServerConfiguration("host123", 1234, 58) + GrpcServerConfiguration("host123", 1234, 58), + GrpcClientConfiguration(400) ) private val RABBITMQ_CONF_JSON = loadConfJson("rabbitMq") diff --git a/src/test/resources/test_json_configurations/grpc.json b/src/test/resources/test_json_configurations/grpc.json index 314389830..5896dc621 100644 --- a/src/test/resources/test_json_configurations/grpc.json +++ b/src/test/resources/test_json_configurations/grpc.json @@ -23,5 +23,8 @@ "host": "host123", "port": 1234, "workers": 58 + }, + "client": { + "keepAliveInterval": 400 } } \ No newline at end of file From e807727c29ed3ab86051a04c79f7c16dfb9a2b74 Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Mon, 5 Sep 2022 07:43:50 +0300 Subject: [PATCH 051/154] Apply suggestions from code review Co-authored-by: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> --- README.md | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9fb65d6fc..b41f67c62 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ dependencies { ## Release notes ### 3.40.1 -+ gRPC configuration now contains `client` property where `keepAliveInterval` can be defined for client grpc services. (default: 300 sec) ++ gRPC configuration now contains `client` property where `keepAliveInterval` can be defined for client grpc services. (default: 60 sec) ### 3.40.0 diff --git a/gradle.properties b/gradle.properties index c63ef9616..3abfd7183 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.40.1 +release_version=3.41.0 description = 'th2 common library (Java)' From 31b264845e47bbebb677cf362b14819b6285b4eb Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Mon, 5 Sep 2022 12:36:33 +0400 Subject: [PATCH 052/154] [TH2-3093] Add option to restrict max message size on interfaces. --- README.md | 2 ++ .../common/schema/grpc/router/impl/DefaultStubStorage.java | 1 + .../th2/common/schema/grpc/configuration/GrpcConfiguration.kt | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b41f67c62..d49794ec2 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,8 @@ The `CommonFactory` reads a grpc's router configuration from the `grpc.json` fil * server - grpc server configuration * client - grpc client configuration * keepAliveInterval - number of seconds before each keep alive message. +* endpoint - grpc endpoint configuration + * maxMessageSize - this option enables endpoint message filtering based on message size (message with size larger than option value will be skipped). By default, it has a value of `4 MB`. The unit of measurement of the value is number of bytes. ```json { diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java index ce4278558..3e59a8c8f 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultStubStorage.java @@ -122,6 +122,7 @@ public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory .keepAliveTime(clientConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .usePlaintext() .intercept(new GrpcInterceptor(service.pinName, methodInvokeCounter, requestBytesCounter, responseBytesCounter)) + .maxInboundMessageSize(endpoint.getMaxMessageSize()) .build(), CallOptions.DEFAULT ); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index f8bcce298..deb817186 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -20,6 +20,7 @@ import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfigura import com.exactpro.th2.common.schema.strategy.route.RoutingStrategy import com.exactpro.th2.service.RetryPolicy import com.fasterxml.jackson.annotation.JsonProperty +import io.grpc.internal.GrpcUtil data class GrpcConfiguration( @JsonProperty var services: Map = emptyMap(), @@ -46,7 +47,8 @@ data class GrpcClientConfiguration( data class GrpcEndpointConfiguration( @JsonProperty(required = true) var host: String, @JsonProperty(required = true) var port: Int = 8080, - var attributes: List = emptyList() + @JsonProperty var maxMessageSize: Int = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE, + var attributes: List = emptyList(), ) : Configuration() data class GrpcRetryConfiguration( From 6d0287348107cc809c5c42ad1bbe42dd50ce54a9 Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Mon, 5 Sep 2022 12:57:05 +0400 Subject: [PATCH 053/154] fix test --- .../com/exactpro/th2/common/schema/TestJsonConfiguration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index fcadeabef..2dab39201 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -138,7 +138,7 @@ class TestJsonConfiguration { init(GrpcRawRobinStrategy(listOf("endpoint"))) }, GrpcConfiguration::class.java, - mapOf("endpoint" to GrpcEndpointConfiguration("host", 12345, listOf("test_attr"))), + mapOf("endpoint" to GrpcEndpointConfiguration("host", 12345, attributes = listOf("test_attr"))), emptyList() ) ), From 1b914a3d3873d206ebb9aef4fa18c7f9f9973198 Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Fri, 9 Sep 2022 15:30:03 +0400 Subject: [PATCH 054/154] limit max message size for server channel --- .../th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java index ca3779d0d..ccfb08b5c 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java @@ -197,6 +197,7 @@ protected Channel getOrCreateChannel(String endpointName, Map.Entry Date: Wed, 14 Sep 2022 13:10:26 +0400 Subject: [PATCH 055/154] move maxMessageSize config option to grpc configuration. --- README.md | 3 +++ .../common/schema/grpc/router/AbstractGrpcRouter.java | 1 + .../schema/grpc/router/impl/DefaultGrpcRouter.java | 4 ++-- .../schema/grpc/router/impl/DefaultStubStorage.java | 9 ++++++--- .../schema/grpc/configuration/GrpcConfiguration.kt | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d49794ec2..4da6eea20 100644 --- a/README.md +++ b/README.md @@ -350,6 +350,9 @@ dependencies { ## Release notes +### 3.40.2 ++ Added gRPC service setting to be able to change maxMessageSize + ### 3.40.1 + gRPC configuration now contains `client` property where `keepAliveInterval` can be defined for client grpc services. (default: 60 sec) diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java index 1bce55a5d..361ee1eea 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java @@ -134,6 +134,7 @@ public Server startServer(BindableService... services) { builder = builder.workerEventLoopGroup(eventLoop) .bossEventLoopGroup(eventLoop) .channelType(NioServerSocketChannel.class) + .maxInboundMessageSize(configuration.getMaxMessageSize()) .intercept(new GrpcInterceptor("server", GRPC_RECEIVE_CALL_TOTAL, GRPC_RECEIVE_CALL_REQUEST_BYTES, GRPC_RECEIVE_CALL_RESPONSE_BYTES)); builder.addService(ProtoReflectionService.newInstance()); diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java index ccfb08b5c..caf1181fb 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java @@ -89,7 +89,7 @@ public T getService(@NotNull Class cls) throws ClassNotFoundException { GRPC_INVOKE_CALL_TOTAL, GRPC_INVOKE_CALL_REQUEST_BYTES, GRPC_INVOKE_CALL_RESPONSE_BYTES, - configuration.getClientConfiguration()) + configuration) ) ); } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { @@ -197,7 +197,7 @@ protected Channel getOrCreateChannel(String endpointName, Map.Entry { private final List> services; private final GrpcClientConfiguration clientConfiguration; + private final GrpcConfiguration configuration; private final Counter methodInvokeCounter; private final Counter requestBytesCounter; private final Counter responseBytesCounter; @@ -69,12 +71,13 @@ public DefaultStubStorage( @NotNull Counter methodInvokeCounter, @NotNull Counter requestBytesCounter, @NotNull Counter responseBytesCounter, - @NotNull GrpcClientConfiguration clientConfiguration + @NotNull GrpcConfiguration configuration ) { this.methodInvokeCounter = methodInvokeCounter; this.requestBytesCounter = requestBytesCounter; this.responseBytesCounter = responseBytesCounter; - this.clientConfiguration = clientConfiguration; + this.clientConfiguration = configuration.getClientConfiguration(); + this.configuration = configuration; services = new ArrayList<>(serviceConfigurations.size()); for (final var config: serviceConfigurations) { @@ -122,7 +125,7 @@ public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory .keepAliveTime(clientConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .usePlaintext() .intercept(new GrpcInterceptor(service.pinName, methodInvokeCounter, requestBytesCounter, responseBytesCounter)) - .maxInboundMessageSize(endpoint.getMaxMessageSize()) + .maxInboundMessageSize(configuration.getMaxMessageSize()) .build(), CallOptions.DEFAULT ); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index deb817186..1eb1a81fe 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -26,6 +26,7 @@ data class GrpcConfiguration( @JsonProperty var services: Map = emptyMap(), @JsonProperty(value = "server") var serverConfiguration: GrpcServerConfiguration = GrpcServerConfiguration(), @JsonProperty(value = "client") var clientConfiguration: GrpcClientConfiguration = GrpcClientConfiguration(), + @JsonProperty var maxMessageSize: Int = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE, @JsonProperty var retryConfiguration: GrpcRetryConfiguration = GrpcRetryConfiguration() ) : Configuration() @@ -47,7 +48,6 @@ data class GrpcClientConfiguration( data class GrpcEndpointConfiguration( @JsonProperty(required = true) var host: String, @JsonProperty(required = true) var port: Int = 8080, - @JsonProperty var maxMessageSize: Int = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE, var attributes: List = emptyList(), ) : Configuration() From 8b57e98e850878b67c4ce42d48ce8b80872f5e8f Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Wed, 14 Sep 2022 13:13:44 +0400 Subject: [PATCH 056/154] update readme --- README.md | 4 ++-- gradle.properties | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4da6eea20..39b9822af 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (3.40.0) +# th2 common library (Java) (3.42.0) ## Usage @@ -350,7 +350,7 @@ dependencies { ## Release notes -### 3.40.2 +### 3.42.0 + Added gRPC service setting to be able to change maxMessageSize ### 3.40.1 diff --git a/gradle.properties b/gradle.properties index 3abfd7183..d9abe0345 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.41.0 +release_version=3.42.0 description = 'th2 common library (Java)' From 3cf9020207e7de98768955230f99b23add5d7951 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 14 Sep 2022 16:21:20 +0400 Subject: [PATCH 057/154] [th2-3093] Corrected place of keepAliveInterval option --- README.md | 13 +++++-------- gradle.properties | 2 +- .../schema/grpc/router/AbstractGrpcRouter.java | 1 + .../schema/grpc/router/impl/DefaultGrpcRouter.java | 2 ++ .../schema/grpc/router/impl/DefaultStubStorage.java | 7 +++---- .../schema/grpc/configuration/GrpcConfiguration.kt | 8 ++------ .../th2/common/schema/TestJsonConfiguration.kt | 3 +-- .../resources/test_json_configurations/grpc.json | 4 +--- 8 files changed, 16 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 39b9822af..fe4a08db2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (3.42.0) +# th2 common library (Java) (3.41.0) ## Usage @@ -163,8 +163,7 @@ Filters format: The `CommonFactory` reads a grpc's router configuration from the `grpc.json` file. * services - grpc services configurations * server - grpc server configuration -* client - grpc client configuration - * keepAliveInterval - number of seconds before each keep alive message. +* keepAliveInterval - number of seconds before each keep alive message. * endpoint - grpc endpoint configuration * maxMessageSize - this option enables endpoint message filtering based on message size (message with size larger than option value will be skipped). By default, it has a value of `4 MB`. The unit of measurement of the value is number of bytes. @@ -196,7 +195,7 @@ The `CommonFactory` reads a grpc's router configuration from the `grpc.json` fil "workers": 58 }, "client": { - "keepAliveInterval": 400 + "keepAliveInterval": 60 } } ``` @@ -350,12 +349,10 @@ dependencies { ## Release notes -### 3.42.0 +### 3.41.0 ++ gRPC's configuration now contains `keepAliveInterval` property. It can be defined for grpc services. (default: 60 sec) + Added gRPC service setting to be able to change maxMessageSize -### 3.40.1 -+ gRPC configuration now contains `client` property where `keepAliveInterval` can be defined for client grpc services. (default: 60 sec) - ### 3.40.0 + gRPC router creates server support [grpc-service-reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md#grpc-server-reflection-tutorial) diff --git a/gradle.properties b/gradle.properties index d9abe0345..3abfd7183 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.42.0 +release_version=3.41.0 description = 'th2 common library (Java)' diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java index 361ee1eea..78169ba57 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java @@ -134,6 +134,7 @@ public Server startServer(BindableService... services) { builder = builder.workerEventLoopGroup(eventLoop) .bossEventLoopGroup(eventLoop) .channelType(NioServerSocketChannel.class) + .keepAliveTime(configuration.getKeepAliveInterval(), TimeUnit.SECONDS) .maxInboundMessageSize(configuration.getMaxMessageSize()) .intercept(new GrpcInterceptor("server", GRPC_RECEIVE_CALL_TOTAL, GRPC_RECEIVE_CALL_REQUEST_BYTES, GRPC_RECEIVE_CALL_RESPONSE_BYTES)); diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java index caf1181fb..cff66448f 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java @@ -42,6 +42,7 @@ import java.util.ServiceLoader; import java.util.ServiceLoader.Provider; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -197,6 +198,7 @@ protected Channel getOrCreateChannel(String endpointName, Map.Entry { } private final List> services; - private final GrpcClientConfiguration clientConfiguration; + private final @NotNull GrpcConfiguration clientConfiguration; private final GrpcConfiguration configuration; private final Counter methodInvokeCounter; private final Counter requestBytesCounter; @@ -76,7 +75,7 @@ public DefaultStubStorage( this.methodInvokeCounter = methodInvokeCounter; this.requestBytesCounter = requestBytesCounter; this.responseBytesCounter = responseBytesCounter; - this.clientConfiguration = configuration.getClientConfiguration(); + this.clientConfiguration = configuration; this.configuration = configuration; services = new ArrayList<>(serviceConfigurations.size()); @@ -122,9 +121,9 @@ public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory return stubFactory.newStub( ManagedChannelBuilder.forAddress(endpoint.getHost(), endpoint.getPort()) - .keepAliveTime(clientConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .usePlaintext() .intercept(new GrpcInterceptor(service.pinName, methodInvokeCounter, requestBytesCounter, responseBytesCounter)) + .keepAliveTime(clientConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .maxInboundMessageSize(configuration.getMaxMessageSize()) .build(), CallOptions.DEFAULT diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index 1eb1a81fe..bfd247b15 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -25,9 +25,9 @@ import io.grpc.internal.GrpcUtil data class GrpcConfiguration( @JsonProperty var services: Map = emptyMap(), @JsonProperty(value = "server") var serverConfiguration: GrpcServerConfiguration = GrpcServerConfiguration(), - @JsonProperty(value = "client") var clientConfiguration: GrpcClientConfiguration = GrpcClientConfiguration(), @JsonProperty var maxMessageSize: Int = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE, - @JsonProperty var retryConfiguration: GrpcRetryConfiguration = GrpcRetryConfiguration() + @JsonProperty var retryConfiguration: GrpcRetryConfiguration = GrpcRetryConfiguration(), + @JsonProperty var keepAliveInterval: Long = 60L ) : Configuration() data class GrpcServiceConfiguration( @@ -41,10 +41,6 @@ data class Filter( @JsonProperty(required = true) var properties: List, ) : Configuration() -data class GrpcClientConfiguration( - @JsonProperty var keepAliveInterval: Long = 60L -) - data class GrpcEndpointConfiguration( @JsonProperty(required = true) var host: String, @JsonProperty(required = true) var port: Int = 8080, diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index 2dab39201..f5884ce55 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -18,7 +18,6 @@ package com.exactpro.th2.common.schema import com.exactpro.th2.common.metrics.PrometheusConfiguration import com.exactpro.th2.common.schema.cradle.CradleConfidentialConfiguration import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration -import com.exactpro.th2.common.schema.grpc.configuration.GrpcClientConfiguration import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration import com.exactpro.th2.common.schema.grpc.configuration.GrpcEndpointConfiguration import com.exactpro.th2.common.schema.grpc.configuration.GrpcRawRobinStrategy @@ -143,7 +142,7 @@ class TestJsonConfiguration { ) ), GrpcServerConfiguration("host123", 1234, 58), - GrpcClientConfiguration(400) + keepAliveInterval = 400 ) private val RABBITMQ_CONF_JSON = loadConfJson("rabbitMq") diff --git a/src/test/resources/test_json_configurations/grpc.json b/src/test/resources/test_json_configurations/grpc.json index 5896dc621..62e30d1a8 100644 --- a/src/test/resources/test_json_configurations/grpc.json +++ b/src/test/resources/test_json_configurations/grpc.json @@ -24,7 +24,5 @@ "port": 1234, "workers": 58 }, - "client": { - "keepAliveInterval": 400 - } + "keepAliveInterval": 400 } \ No newline at end of file From f1ed46972cd1f6cbd1a050afaa885ecf25de7076 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 14 Sep 2022 16:49:20 +0400 Subject: [PATCH 058/154] [th2-3093] Updated th2-bom form 3.2.0 to 3.3.0-SNAPSHOT --- README.md | 1 + build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fe4a08db2..018b2dcfc 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,7 @@ dependencies { ### 3.41.0 + gRPC's configuration now contains `keepAliveInterval` property. It can be defined for grpc services. (default: 60 sec) + Added gRPC service setting to be able to change maxMessageSize ++ Updated th2-bom form 3.2.0 to 3.3.0 ### 3.40.0 diff --git a/build.gradle b/build.gradle index 007650c22..2c26cad32 100644 --- a/build.gradle +++ b/build.gradle @@ -166,7 +166,7 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:3.2.0") + api platform("com.exactpro.th2:bom:3.3.0-TH2-3093-3052871635-SNAPSHOT") api ("com.exactpro.th2:cradle-core:${cradleVersion}") api 'com.exactpro.th2:grpc-common:3.11.1' @@ -178,7 +178,7 @@ dependencies { implementation "io.grpc:grpc-protobuf" implementation "io.grpc:grpc-core" implementation "io.grpc:grpc-netty" - implementation "io.grpc:grpc-services:1.32.1" //FIXME: add to bom + implementation "io.grpc:grpc-services:1.49.0" //FIXME: add to bom implementation "com.rabbitmq:amqp-client" From d11178ef654db8fca7d47ae3e1ad668f92e07e43 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Fri, 23 Sep 2022 12:38:40 +0400 Subject: [PATCH 059/154] [TH2-4188] Refactored GrpcInterceptors.kt * Added enableSizeMeasuring option into grpc_router config --- README.md | 20 +++- .../grpc/router/AbstractGrpcRouter.java | 56 +++++++-- .../grpc/router/impl/DefaultGrpcRouter.java | 18 ++- .../grpc/router/impl/DefaultStubStorage.java | 74 +++++++----- .../grpc/router/AbstractGrpcInterceptor.kt | 52 ++++++++ .../grpc/router/ClientGrpcInterceptor.kt | 70 +++++++++++ .../common/grpc/router/GrpcInterceptors.kt | 112 ------------------ .../th2/common/grpc/router/MethodDetails.kt | 19 +++ .../grpc/router/ServerGrpcInterceptor.kt | 67 +++++++++++ .../grpc/configuration/GrpcConfiguration.kt | 1 - .../configuration/GrpcRouterConfiguration.kt | 5 +- .../common/schema/TestJsonConfiguration.kt | 1 - 12 files changed, 327 insertions(+), 168 deletions(-) create mode 100644 src/main/kotlin/com/exactpro/th2/common/grpc/router/AbstractGrpcInterceptor.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/grpc/router/ClientGrpcInterceptor.kt delete mode 100644 src/main/kotlin/com/exactpro/th2/common/grpc/router/GrpcInterceptors.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/grpc/router/MethodDetails.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/grpc/router/ServerGrpcInterceptor.kt diff --git a/README.md b/README.md index c67272a88..ec74ad870 100644 --- a/README.md +++ b/README.md @@ -160,10 +160,20 @@ Filters format: } ``` -The `CommonFactory` reads a grpc's router configuration from the `grpc.json` file. +The `CommonFactory` reads a grpc's router configuration from the `grpc_router.json` file. +* enableSizeMeasuring - this option enables the gRPC message size measuring. Please note the feature decrease gRPC throughput. +* keepAliveInterval - number of seconds before each keep alive message. + +```json +{ + "enableSizeMeasuring": false, + "keepAliveInterval": 60 +} +``` + +The `CommonFactory` reads a grpc's configuration from the `grpc.json` file. * services - grpc services configurations * server - grpc server configuration -* keepAliveInterval - number of seconds before each keep alive message. * endpoint - grpc endpoint configuration * maxMessageSize - this option enables endpoint message filtering based on message size (message with size larger than option value will be skipped). By default, it has a value of `4 MB`. The unit of measurement of the value is number of bytes. @@ -193,9 +203,6 @@ The `CommonFactory` reads a grpc's router configuration from the `grpc.json` fil "host": "host123", "port": 1234, "workers": 58 - }, - "client": { - "keepAliveInterval": 60 } } ``` @@ -350,7 +357,8 @@ dependencies { ## Release notes ### 3.41.0 -+ gRPC's configuration now contains `keepAliveInterval` property. It can be defined for grpc services. (default: 60 sec) ++ Added the `enableSizeMeasuring` option into gRPC's router configuration. Default value is false ++ gRPC's router configuration now contains `keepAliveInterval` property. It can be defined for grpc services. (default: 60 sec) + Added gRPC service setting to be able to change maxMessageSize + Updated th2-bom form 3.2.0 to 3.3.0 + Started using cradle 3.2, version with grouped messages diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java index 78169ba57..7ca26d2b4 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java @@ -15,7 +15,8 @@ package com.exactpro.th2.common.schema.grpc.router; -import com.exactpro.th2.common.grpc.router.GrpcInterceptor; +import com.exactpro.th2.common.grpc.router.ServerGrpcInterceptor; +import com.exactpro.th2.common.grpc.router.MethodDetails; import com.exactpro.th2.common.metrics.CommonMetrics; import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration; import com.exactpro.th2.common.schema.grpc.configuration.GrpcRouterConfiguration; @@ -36,11 +37,14 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.ExecutorService; +import java.util.function.Function; /** * Abstract implementation for {@link GrpcRouter} @@ -61,6 +65,7 @@ public abstract class AbstractGrpcRouter implements GrpcRouter { protected final List servers = new ArrayList<>(); protected final List loopGroups = new ArrayList<>(); protected final List executors = new ArrayList<>(); + protected GrpcRouterConfiguration routerConfiguration; protected GrpcConfiguration configuration; protected static final Counter GRPC_INVOKE_CALL_TOTAL = Counter.build() @@ -69,36 +74,48 @@ public abstract class AbstractGrpcRouter implements GrpcRouter { .help("Total number of calling particular gRPC method") .register(); + protected static final Map GRPC_INVOKE_CALL_MAP = new ConcurrentHashMap<>(); + protected static final Counter GRPC_INVOKE_CALL_REQUEST_BYTES = Counter.build() .name("th2_grpc_invoke_call_request_bytes") .labelNames(CommonMetrics.TH2_PIN_LABEL, CommonMetrics.GRPC_SERVICE_NAME_LABEL, CommonMetrics.GRPC_METHOD_NAME_LABEL) .help("Number of bytes sent to particular gRPC call") .register(); + protected static final Map GRPC_INVOKE_CALL_REQUEST_SIZE_MAP = new ConcurrentHashMap<>(); + protected static final Counter GRPC_INVOKE_CALL_RESPONSE_BYTES = Counter.build() .name("th2_grpc_invoke_call_response_bytes") .labelNames(CommonMetrics.TH2_PIN_LABEL, CommonMetrics.GRPC_SERVICE_NAME_LABEL, CommonMetrics.GRPC_METHOD_NAME_LABEL) .help("Number of bytes sent to particular gRPC call") .register(); + protected static final Map GRPC_INVOKE_CALL_RESPONSE_SIZE_MAP = new ConcurrentHashMap<>(); + protected static final Counter GRPC_RECEIVE_CALL_TOTAL = Counter.build() .name("th2_grpc_receive_call_total") .labelNames(CommonMetrics.TH2_PIN_LABEL, CommonMetrics.GRPC_SERVICE_NAME_LABEL, CommonMetrics.GRPC_METHOD_NAME_LABEL) .help("Total number of consuming particular gRPC method") .register(); + protected static final Map GRPC_RECEIVE_CALL_MAP = new ConcurrentHashMap<>(); + protected static final Counter GRPC_RECEIVE_CALL_REQUEST_BYTES = Counter.build() .name("th2_grpc_receive_call_request_bytes") .labelNames(CommonMetrics.TH2_PIN_LABEL, CommonMetrics.GRPC_SERVICE_NAME_LABEL, CommonMetrics.GRPC_METHOD_NAME_LABEL) .help("Number of bytes received from particular gRPC call") .register(); + protected static final Map GRPC_RECEIVE_CALL_REQUEST_SIZE_MAP = new ConcurrentHashMap<>(); + protected static final Counter GRPC_RECEIVE_CALL_RESPONSE_BYTES = Counter.build() .name("th2_grpc_receive_call_response_bytes") .labelNames(CommonMetrics.TH2_PIN_LABEL, CommonMetrics.GRPC_SERVICE_NAME_LABEL, CommonMetrics.GRPC_METHOD_NAME_LABEL) .help("Number of bytes sent to particular gRPC call") .register(); + protected static final Map GRPC_RECEIVE_CALL_RESPONSE_SIZE_MAP = new ConcurrentHashMap<>(); + @Override public void init(GrpcRouterConfiguration configuration) { init(new GrpcConfiguration(), configuration); @@ -108,6 +125,7 @@ public void init(GrpcRouterConfiguration configuration) { public void init(@NotNull GrpcConfiguration configuration, @NotNull GrpcRouterConfiguration routerConfiguration) { throwIsInit(); + this.routerConfiguration = Objects.requireNonNull(routerConfiguration); this.configuration = Objects.requireNonNull(configuration); } @@ -134,9 +152,13 @@ public Server startServer(BindableService... services) { builder = builder.workerEventLoopGroup(eventLoop) .bossEventLoopGroup(eventLoop) .channelType(NioServerSocketChannel.class) - .keepAliveTime(configuration.getKeepAliveInterval(), TimeUnit.SECONDS) + .keepAliveTime(routerConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .maxInboundMessageSize(configuration.getMaxMessageSize()) - .intercept(new GrpcInterceptor("server", GRPC_RECEIVE_CALL_TOTAL, GRPC_RECEIVE_CALL_REQUEST_BYTES, GRPC_RECEIVE_CALL_RESPONSE_BYTES)); + .intercept(new ServerGrpcInterceptor("server", + createGetMetric(GRPC_INVOKE_CALL_TOTAL, GRPC_INVOKE_CALL_MAP), + createGetMetric(GRPC_RECEIVE_CALL_TOTAL, GRPC_RECEIVE_CALL_MAP), + createGetMeasuringMetric(GRPC_RECEIVE_CALL_REQUEST_BYTES, GRPC_RECEIVE_CALL_REQUEST_SIZE_MAP), + createGetMeasuringMetric(GRPC_RECEIVE_CALL_RESPONSE_BYTES, GRPC_RECEIVE_CALL_RESPONSE_SIZE_MAP))); builder.addService(ProtoReflectionService.newInstance()); @@ -153,12 +175,6 @@ public Server startServer(BindableService... services) { return server; } - protected void throwIsInit() { - if (this.configuration != null) { - throw new IllegalStateException("Grpc router already init"); - } - } - @Override public void close() { for (Server server : servers) { @@ -196,4 +212,26 @@ public void close() { } }); } + + protected Function createGetMetric(Counter counter, Map map) { + return (MethodDetails methodDetails) -> map + .computeIfAbsent(methodDetails, + (key) -> counter + .labels(methodDetails.getPinName(), + methodDetails.getServiceName(), + methodDetails.getMethodName())); + } + + protected Function createGetMeasuringMetric(Counter counter, Map map) { + if (routerConfiguration.getEnableSizeMeasuring()) { + return createGetMetric(counter, map); + } + return (MethodDetails) -> null; + } + + protected void throwIsInit() { + if (this.configuration != null) { + throw new IllegalStateException("Grpc router already init"); + } + } } \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java index cff66448f..a1c3b7da5 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java @@ -15,7 +15,7 @@ package com.exactpro.th2.common.schema.grpc.router.impl; -import com.exactpro.th2.common.grpc.router.GrpcInterceptor; +import com.exactpro.th2.common.grpc.router.ClientGrpcInterceptor; import com.exactpro.th2.common.schema.exception.InitGrpcRouterException; import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration; import com.exactpro.th2.common.schema.grpc.configuration.GrpcServiceConfiguration; @@ -87,9 +87,11 @@ public T getService(@NotNull Class cls) throws ClassNotFoundException { configuration.getRetryConfiguration(), stubsStorages.computeIfAbsent(cls, key -> new DefaultStubStorage<>(getServiceConfig(key), - GRPC_INVOKE_CALL_TOTAL, - GRPC_INVOKE_CALL_REQUEST_BYTES, - GRPC_INVOKE_CALL_RESPONSE_BYTES, + createGetMetric(GRPC_INVOKE_CALL_TOTAL, GRPC_INVOKE_CALL_MAP), + createGetMetric(GRPC_RECEIVE_CALL_TOTAL, GRPC_RECEIVE_CALL_MAP), + createGetMeasuringMetric(GRPC_INVOKE_CALL_REQUEST_BYTES, GRPC_INVOKE_CALL_REQUEST_SIZE_MAP), + createGetMeasuringMetric(GRPC_INVOKE_CALL_RESPONSE_BYTES, GRPC_INVOKE_CALL_RESPONSE_SIZE_MAP), + routerConfiguration, configuration) ) ); @@ -197,8 +199,12 @@ protected Channel getOrCreateChannel(String endpointName, Map.Entry> implements StubStorage { @@ -58,25 +60,29 @@ private static class ServiceHolder { } private final List> services; - private final @NotNull GrpcConfiguration clientConfiguration; - private final GrpcConfiguration configuration; - private final Counter methodInvokeCounter; - private final Counter requestBytesCounter; - private final Counter responseBytesCounter; + private final @NotNull GrpcRouterConfiguration routerConfiguration; + private final @NotNull GrpcConfiguration configuration; + private final @NotNull Function methodInvokeCounter; + private final @NotNull Function methodReceiveCounter; + private final @NotNull Function requestBytesCounter; + private final @NotNull Function responseBytesCounter; public DefaultStubStorage( @NotNull List> serviceConfigurations, - @NotNull Counter methodInvokeCounter, - @NotNull Counter requestBytesCounter, - @NotNull Counter responseBytesCounter, + @NotNull Function methodInvokeCounter, + @NotNull Function methodReceiveCounter, + @NotNull Function requestBytesCounter, + @NotNull Function responseBytesCounter, + @NotNull GrpcRouterConfiguration routerConfiguration, @NotNull GrpcConfiguration configuration ) { - this.methodInvokeCounter = methodInvokeCounter; - this.requestBytesCounter = requestBytesCounter; - this.responseBytesCounter = responseBytesCounter; - this.clientConfiguration = configuration; - this.configuration = configuration; + this.methodInvokeCounter = requireNonNull(methodInvokeCounter); + this.methodReceiveCounter = requireNonNull(methodReceiveCounter); + this.requestBytesCounter = requireNonNull(requestBytesCounter); + this.responseBytesCounter = requireNonNull(responseBytesCounter); + this.routerConfiguration = requireNonNull(routerConfiguration); + this.configuration = requireNonNull(configuration); services = new ArrayList<>(serviceConfigurations.size()); for (final var config: serviceConfigurations) { @@ -122,8 +128,12 @@ public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory return stubFactory.newStub( ManagedChannelBuilder.forAddress(endpoint.getHost(), endpoint.getPort()) .usePlaintext() - .intercept(new GrpcInterceptor(service.pinName, methodInvokeCounter, requestBytesCounter, responseBytesCounter)) - .keepAliveTime(clientConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) + .intercept(new ClientGrpcInterceptor(service.pinName, + methodInvokeCounter, + methodReceiveCounter, + requestBytesCounter, + responseBytesCounter)) + .keepAliveTime(routerConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .maxInboundMessageSize(configuration.getMaxMessageSize()) .build(), CallOptions.DEFAULT diff --git a/src/main/kotlin/com/exactpro/th2/common/grpc/router/AbstractGrpcInterceptor.kt b/src/main/kotlin/com/exactpro/th2/common/grpc/router/AbstractGrpcInterceptor.kt new file mode 100644 index 000000000..167c065a5 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/grpc/router/AbstractGrpcInterceptor.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.grpc.router + +import com.exactpro.th2.common.value.toValue +import io.grpc.MethodDescriptor +import io.prometheus.client.Counter +import mu.KotlinLogging + +abstract class AbstractGrpcInterceptor ( + protected val pinName: String, +) { + companion object { + private val LOGGER = KotlinLogging.logger {} + + fun MethodDescriptor<*, *>.toMethodDetails(pinName: String): MethodDetails { + val idx = fullMethodName.lastIndexOf('/').coerceAtLeast(0) + return MethodDetails( + pinName, + fullMethodName.substring(0, idx), + fullMethodName.substring(idx + 1)) + } + + fun Any.publishMetrics( + text: String, + sizeBytesCounter: Counter.Child?, + methodInvokeCounter: Counter.Child, + ) { + val size: Int? = sizeBytesCounter?.let { + toValue().serializedSize.also { + sizeBytesCounter.inc(it.toDouble()) + } + } + + LOGGER.debug { "$text. ${if (size != null) "Message size = $size" else ""}" } + methodInvokeCounter.inc() + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/grpc/router/ClientGrpcInterceptor.kt b/src/main/kotlin/com/exactpro/th2/common/grpc/router/ClientGrpcInterceptor.kt new file mode 100644 index 000000000..089cb2d0b --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/grpc/router/ClientGrpcInterceptor.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.grpc.router + +import io.grpc.CallOptions +import io.grpc.Channel +import io.grpc.ClientCall +import io.grpc.ClientInterceptor +import io.grpc.ForwardingClientCall +import io.grpc.ForwardingClientCallListener +import io.grpc.Metadata +import io.grpc.MethodDescriptor +import io.prometheus.client.Counter +import java.util.function.Function + +class ClientGrpcInterceptor( + pinName: String, + private val methodInvokeFunc: Function, + private val methodReceiveFunc: Function, + private val requestSizeFunc: Function, + private val responseSizeFunc: Function, +) : AbstractGrpcInterceptor(pinName), ClientInterceptor { + override fun interceptCall( + method: MethodDescriptor, + callOptions: CallOptions, + next: Channel + ): ClientCall { + + val methodDetails = method.toMethodDetails(pinName) + + return object : ForwardingClientCall.SimpleForwardingClientCall(next.newCall(method, callOptions)) { + override fun sendMessage(message: ReqT) { + message.publishMetrics( + "gRPC call ${methodDetails.serviceName}/${methodDetails.methodName} invoked on pin '$pinName'", + requestSizeFunc.apply(methodDetails), + methodInvokeFunc.apply(methodDetails) + ) + super.sendMessage(message) + } + + override fun start(responseListener: Listener?, headers: Metadata?) { + val forwardingListener = + object : ForwardingClientCallListener.SimpleForwardingClientCallListener(responseListener) { + override fun onMessage(message: RespT) { + message.publishMetrics( + "gRPC ${methodDetails.serviceName}/${methodDetails.methodName} response received on pin '$pinName'", + responseSizeFunc.apply(methodDetails), + methodReceiveFunc.apply(methodDetails) + ) + super.onMessage(message) + } + } + super.start(forwardingListener, headers) + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/grpc/router/GrpcInterceptors.kt b/src/main/kotlin/com/exactpro/th2/common/grpc/router/GrpcInterceptors.kt deleted file mode 100644 index be63c118a..000000000 --- a/src/main/kotlin/com/exactpro/th2/common/grpc/router/GrpcInterceptors.kt +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.exactpro.th2.common.grpc.router - -import com.exactpro.th2.common.value.toValue -import io.grpc.CallOptions -import io.grpc.Channel -import io.grpc.ClientCall -import io.grpc.ClientInterceptor -import io.grpc.ForwardingClientCall -import io.grpc.ForwardingClientCallListener -import io.grpc.ForwardingServerCall -import io.grpc.ForwardingServerCallListener -import io.grpc.Metadata -import io.grpc.MethodDescriptor -import io.grpc.ServerCall -import io.grpc.ServerCallHandler -import io.grpc.ServerInterceptor -import io.prometheus.client.Counter -import mu.KotlinLogging - -class GrpcInterceptor( - private val pinName: String, - private val methodInvokeCounter: Counter, - private val requestBytesCounter: Counter, - private val responseBytesCounter: Counter -) : ClientInterceptor, ServerInterceptor { - override fun interceptCall( - method: MethodDescriptor, - callOptions: CallOptions, - next: Channel - ): ClientCall { - - val fullName = method.fullMethodName - val idx = fullName.lastIndexOf('/').coerceAtLeast(0) - val serviceName = fullName.substring(0, idx) - val methodName = fullName.substring(idx + 1) - - return object : ForwardingClientCall.SimpleForwardingClientCall(next.newCall(method, callOptions)) { - override fun sendMessage(message: ReqT) { - val size = message.toValue().serializedSize - LOGGER.debug { "gRPC call $serviceName/$methodName invoked on pin '$pinName'. Message size = $size" } - methodInvokeCounter.labels(pinName, serviceName, methodName).inc() - requestBytesCounter.labels(pinName, serviceName, methodName).inc(size.toDouble()) - super.sendMessage(message) - } - - override fun start(responseListener: Listener?, headers: Metadata?) { - val forwardingListener = - object : ForwardingClientCallListener.SimpleForwardingClientCallListener(responseListener) { - override fun onMessage(message: RespT) { - val size = message.toValue().serializedSize - LOGGER.debug { "gRPC $serviceName/$methodName response received on pin '$pinName'. Message size = $size" } - responseBytesCounter.labels(pinName, serviceName, methodName).inc(size.toDouble()) - super.onMessage(message) - } - } - super.start(forwardingListener, headers) - } - } - } - - override fun interceptCall( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler - ): ServerCall.Listener { - - val fullName = call.methodDescriptor.fullMethodName - val idx = fullName.lastIndexOf('/').coerceAtLeast(0) - val serviceName = fullName.substring(0, idx) - val methodName = fullName.substring(idx + 1) - - val forwardingCall = object : ForwardingServerCall.SimpleForwardingServerCall(call) { - override fun sendMessage(message: RespT) { - val size = message.toValue().serializedSize - LOGGER.debug { "gRPC $serviceName/$methodName response message sent on '$pinName'. Message size = $size" } - responseBytesCounter.labels(pinName, serviceName, methodName).inc(size.toDouble()) - super.sendMessage(message) - } - } - - val listener = next.startCall(forwardingCall, headers) - - return object : ForwardingServerCallListener.SimpleForwardingServerCallListener(listener) { - override fun onMessage(message: ReqT) { - val size = message.toValue().serializedSize - LOGGER.debug { "gRPC call received on '$pinName': $serviceName/$methodName. Message size = $size" } - methodInvokeCounter.labels(pinName, serviceName, methodName).inc() - requestBytesCounter.labels(pinName, serviceName, methodName).inc(size.toDouble()) - super.onMessage(message) - } - } - } - - companion object { - val LOGGER = KotlinLogging.logger {} - } -} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/grpc/router/MethodDetails.kt b/src/main/kotlin/com/exactpro/th2/common/grpc/router/MethodDetails.kt new file mode 100644 index 000000000..d49c0185a --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/grpc/router/MethodDetails.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.grpc.router + +data class MethodDetails(val pinName: String, val serviceName: String, val methodName: String) \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/grpc/router/ServerGrpcInterceptor.kt b/src/main/kotlin/com/exactpro/th2/common/grpc/router/ServerGrpcInterceptor.kt new file mode 100644 index 000000000..e5884c391 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/grpc/router/ServerGrpcInterceptor.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.grpc.router + +import io.grpc.ForwardingServerCall +import io.grpc.ForwardingServerCallListener +import io.grpc.Metadata +import io.grpc.ServerCall +import io.grpc.ServerCallHandler +import io.grpc.ServerInterceptor +import io.prometheus.client.Counter +import java.util.function.Function + +class ServerGrpcInterceptor ( + pinName: String, + private val methodInvokeFunc: Function, + private val methodReceiveFunc: Function, + private val requestSizeFunc: Function, + private val responseSizeFunc: Function, +) : AbstractGrpcInterceptor(pinName), ServerInterceptor { + + override fun interceptCall( + call: ServerCall, + headers: Metadata, + next: ServerCallHandler + ): ServerCall.Listener { + + val methodDetails = call.methodDescriptor.toMethodDetails(pinName) + + val forwardingCall = object : ForwardingServerCall.SimpleForwardingServerCall(call) { + override fun sendMessage(message: RespT) { + message.publishMetrics( + "gRPC ${methodDetails.serviceName}/${methodDetails.methodName} response message sent on '$pinName'", + responseSizeFunc.apply(methodDetails), + methodInvokeFunc.apply(methodDetails) + ) + super.sendMessage(message) + } + } + + val listener = next.startCall(forwardingCall, headers) + + return object : ForwardingServerCallListener.SimpleForwardingServerCallListener(listener) { + override fun onMessage(message: ReqT) { + message.publishMetrics( + "gRPC call received on '$pinName': ${methodDetails.serviceName}/${methodDetails.methodName}", + requestSizeFunc.apply(methodDetails), + methodReceiveFunc.apply(methodDetails) + ) + super.onMessage(message) + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index bfd247b15..e4c407d1e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -27,7 +27,6 @@ data class GrpcConfiguration( @JsonProperty(value = "server") var serverConfiguration: GrpcServerConfiguration = GrpcServerConfiguration(), @JsonProperty var maxMessageSize: Int = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE, @JsonProperty var retryConfiguration: GrpcRetryConfiguration = GrpcRetryConfiguration(), - @JsonProperty var keepAliveInterval: Long = 60L ) : Configuration() data class GrpcServiceConfiguration( diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcRouterConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcRouterConfiguration.kt index 0e1a55ce2..c05f8d65d 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcRouterConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcRouterConfiguration.kt @@ -17,4 +17,7 @@ package com.exactpro.th2.common.schema.grpc.configuration import com.exactpro.th2.common.schema.configuration.Configuration -data class GrpcRouterConfiguration(var workers: Int = 1) : Configuration() \ No newline at end of file +data class GrpcRouterConfiguration( + var enableSizeMeasuring: Boolean = false, + var keepAliveInterval: Long = 60L +) : Configuration() \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index f5884ce55..001d51e51 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -142,7 +142,6 @@ class TestJsonConfiguration { ) ), GrpcServerConfiguration("host123", 1234, 58), - keepAliveInterval = 400 ) private val RABBITMQ_CONF_JSON = loadConfJson("rabbitMq") From 2418cd7d84d656f67b00be34b15a5ff631157b1f Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 27 Sep 2022 11:06:49 +0400 Subject: [PATCH 060/154] [TH2-4188] Added logging for gRPC stubs --- .../th2/common/schema/grpc/router/AbstractGrpcRouter.java | 2 ++ .../common/schema/grpc/router/impl/DefaultGrpcRouter.java | 2 ++ .../common/schema/grpc/router/impl/DefaultStubStorage.java | 6 ++++++ .../common/schema/grpc/configuration/GrpcConfiguration.kt | 2 +- 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java index 7ca26d2b4..555e04ca6 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java @@ -172,6 +172,8 @@ public Server startServer(BindableService... services) { loopGroups.add(eventLoop); servers.add(server); + LOGGER.info("Made gRPC server: host {}, port {}, keepAliveTime {}, max inbound message {}", serverConf.getHost(), serverConf.getPort(), routerConfiguration.getKeepAliveInterval(), configuration.getMaxMessageSize()); + return server; } diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java index a1c3b7da5..546885f11 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java @@ -198,6 +198,8 @@ protected Channel getOrCreateChannel(String endpointName, Map.Entry> implements StubStorage { + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultStubStorage.class); + private static class ServiceHolder { final String pinName; GrpcServiceConfiguration serviceConfig; @@ -125,6 +129,8 @@ public T getStub(@NotNull Message message, @NotNull AbstractStub.StubFactory "that matches the provided alias: " + key); } + LOGGER.info("Made gRPC stub: host {}, port {}, keepAliveTime {}, max inbound message {}", endpoint.getHost(), endpoint.getPort(), routerConfiguration.getKeepAliveInterval(), configuration.getMaxMessageSize()); + return stubFactory.newStub( ManagedChannelBuilder.forAddress(endpoint.getHost(), endpoint.getPort()) .usePlaintext() diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt index e4c407d1e..958edc756 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcConfiguration.kt @@ -63,5 +63,5 @@ data class GrpcRetryConfiguration( data class GrpcServerConfiguration( var host: String? = "localhost", @JsonProperty(required = true) var port: Int = 8080, - var workers: Int = 1 + var workers: Int = 5 ) : Configuration() \ No newline at end of file From bb2dedb3bf9b0c92916457c6df6c9c24247054ff Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 27 Sep 2022 11:34:42 +0400 Subject: [PATCH 061/154] [TH2-4188] set keepAliveTimeout instead of keepAliveTime for gRPC --- .../th2/common/schema/grpc/router/AbstractGrpcRouter.java | 2 +- .../th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java | 2 +- .../th2/common/schema/grpc/router/impl/DefaultStubStorage.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java index 555e04ca6..fb67dee8c 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java @@ -152,7 +152,7 @@ public Server startServer(BindableService... services) { builder = builder.workerEventLoopGroup(eventLoop) .bossEventLoopGroup(eventLoop) .channelType(NioServerSocketChannel.class) - .keepAliveTime(routerConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) + .keepAliveTimeout(routerConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .maxInboundMessageSize(configuration.getMaxMessageSize()) .intercept(new ServerGrpcInterceptor("server", createGetMetric(GRPC_INVOKE_CALL_TOTAL, GRPC_INVOKE_CALL_MAP), diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java index 546885f11..d4b3547c4 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/impl/DefaultGrpcRouter.java @@ -206,7 +206,7 @@ protected Channel getOrCreateChannel(String endpointName, Map.Entry methodReceiveCounter, requestBytesCounter, responseBytesCounter)) - .keepAliveTime(routerConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) + .keepAliveTimeout(routerConfiguration.getKeepAliveInterval(), TimeUnit.SECONDS) .maxInboundMessageSize(configuration.getMaxMessageSize()) .build(), CallOptions.DEFAULT From cb02f5ec099815c50456f1af90f91a8a52642218 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 20 Oct 2022 11:14:37 +0400 Subject: [PATCH 062/154] [TH2-4299] added isRedelivered flag, added reject() along with existing confirm() --- .../schema/message/MessageListener.java | 8 ++ .../rabbitmq/AbstractRabbitSubscriber.java | 13 +-- .../connection/ConnectionManager.java | 97 ++++++++++++------- .../message/ConfirmationMessageListener.kt | 9 ++ .../common/schema/message/DeliveryMetadata.kt | 21 ++++ .../message/ManualAckDeliveryCallback.kt | 13 +-- .../message/impl/OnlyOnceConfirmation.kt | 10 +- 7 files changed, 119 insertions(+), 52 deletions(-) create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java index 6cbc0570d..4f312e527 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java @@ -21,8 +21,16 @@ */ public interface MessageListener { + @Deprecated( + since = "This method does not provide all necessary information about a message" + ) void handle(String consumerTag, T message) throws Exception; + default void handle(DeliveryMetadata deliveryMetadata, T message) throws Exception { + handle(deliveryMetadata.getConsumerTag(), message); + } + default void onClose() {} + } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index 3941c4c12..dd1284cd3 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -17,6 +17,7 @@ import com.exactpro.th2.common.metrics.HealthMetrics; import com.exactpro.th2.common.schema.message.ConfirmationMessageListener; +import com.exactpro.th2.common.schema.message.DeliveryMetadata; import com.exactpro.th2.common.schema.message.FilterFunction; import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.exactpro.th2.common.schema.message.MessageSubscriber; @@ -116,7 +117,7 @@ public void start() throws Exception { try { monitor = connectionManager.basicConsume( queue, - (consumeTag, delivery, confirmation) -> { + (deliveryMetadata, delivery, confirmProcessed) -> { Timer processTimer = MESSAGE_PROCESS_DURATION_SECONDS .labels(th2Pin, th2Type, queue) .startTimer(); @@ -128,8 +129,7 @@ public void start() throws Exception { try { value = valueFromBytes(delivery.getBody()); } catch (Exception e) { - LOGGER.error("Couldn't parse delivery. Confirm message received", e); - confirmation.confirm(); + confirmProcessed.reject(); throw new IOException( String.format( "Can not extract value from bytes for envelope '%s', queue '%s', pin '%s'", @@ -138,7 +138,7 @@ public void start() throws Exception { e ); } - handle(consumeTag, delivery, value, confirmation); + handle(deliveryMetadata.getConsumerTag(), delivery, value, confirmProcessed); } finally { processTimer.observeDuration(); } @@ -225,7 +225,8 @@ protected void handle(String consumeTag, Delivery delivery, T value, Confirmatio boolean hasManualConfirmation = false; for (ConfirmationMessageListener listener : listeners) { try { - listener.handle(consumeTag, filteredValue, confirmation); + boolean redeliver = delivery.getEnvelope().isRedeliver(); + listener.handle(new DeliveryMetadata(consumeTag, redeliver), filteredValue, confirmation); if (!hasManualConfirmation) { hasManualConfirmation = ConfirmationMessageListener.isManual(listener); } @@ -239,7 +240,7 @@ protected void handle(String consumeTag, Delivery delivery, T value, Confirmatio } catch (Exception e) { LOGGER.error("Can not parse value from delivery for: {}", consumeTag, e); try { - confirmation.confirm(); + confirmation.reject(); } catch (IOException ex) { LOGGER.error("Cannot confirm delivery for {}", consumeTag, ex); } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java index f9eeba828..df1f0fe31 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java @@ -14,37 +14,8 @@ */ package com.exactpro.th2.common.schema.message.impl.rabbitmq.connection; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.BiConsumer; -import java.util.function.Supplier; - -import javax.annotation.concurrent.GuardedBy; - -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.commons.lang3.builder.ToStringBuilder; -import org.apache.commons.lang3.builder.ToStringStyle; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.exactpro.th2.common.metrics.HealthMetrics; +import com.exactpro.th2.common.schema.message.DeliveryMetadata; import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback; import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.exactpro.th2.common.schema.message.SubscriberMonitor; @@ -65,6 +36,34 @@ import com.rabbitmq.client.RecoveryListener; import com.rabbitmq.client.ShutdownNotifier; import com.rabbitmq.client.TopologyRecoveryException; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.concurrent.GuardedBy; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; +import java.util.function.Supplier; public class ConnectionManager implements AutoCloseable { public static final String EMPTY_ROUTING_KEY = ""; @@ -299,13 +298,32 @@ public SubscriberMonitor basicConsume(String queue, ManualAckDeliveryCallback de String routingKey = envelope.getRoutingKey(); LOGGER.trace("Received delivery {} from queue={} routing_key={}", deliveryTag, queue, routingKey); - Confirmation confirmation = OnlyOnceConfirmation.wrap("from " + routingKey + " to " + queue, () -> holder.withLock(ch -> { - try { - basicAck(ch, deliveryTag); - } finally { - holder.release(() -> metrics.getReadinessMonitor().enable()); + Confirmation wrappedConfirmation = new Confirmation() { + @Override + public void reject() throws IOException { + holder.withLock(ch -> { + try { + basicReject(ch, deliveryTag); + } finally { + holder.release(() -> metrics.getReadinessMonitor().enable()); + } + }); + } + + @Override + public void confirm() throws IOException { + holder.withLock(ch -> { + try { + basicAck(ch, deliveryTag); + } finally { + holder.release(() -> metrics.getReadinessMonitor().enable()); + } + }); } - })); + }; + + Confirmation confirmation = OnlyOnceConfirmation.wrap("from " + routingKey + " to " + queue, wrappedConfirmation); + holder.withLock(() -> holder.acquireAndSubmitCheck(() -> channelChecker.schedule(() -> { @@ -319,7 +337,8 @@ public SubscriberMonitor basicConsume(String queue, ManualAckDeliveryCallback de return false; // to cast to Callable }, configuration.getConfirmationTimeout().toMillis(), TimeUnit.MILLISECONDS) )); - deliverCallback.handle(tagTmp, delivery, confirmation); + boolean redeliver = envelope.isRedeliver(); + deliverCallback.handle(new DeliveryMetadata(tagTmp, redeliver), delivery, confirmation); } catch (IOException | RuntimeException e) { LOGGER.error("Cannot handle delivery for tag {}: {}", tagTmp, e.getMessage(), e); } @@ -430,6 +449,10 @@ private static void basicAck(Channel channel, long deliveryTag) throws IOExcepti channel.basicAck(deliveryTag, false); } + private static void basicReject(Channel channel, long deliveryTag) throws IOException { + channel.basicReject(deliveryTag, false); + } + private static class RabbitMqSubscriberMonitor implements SubscriberMonitor { private final ChannelHolder holder; diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt index 4fc6eceff..2c31cbb3a 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt @@ -19,8 +19,17 @@ package com.exactpro.th2.common.schema.message interface ConfirmationMessageListener { @Throws(Exception::class) + @Deprecated( + "This method does not provide all necessary information about a message", + ReplaceWith("handle(deliveryMetadata, message, confirmation)") + ) fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) + @Throws(Exception::class) + fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { + handle(deliveryMetadata.consumerTag, message, confirmation) + } + fun onClose() {} companion object { diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt new file mode 100644 index 000000000..980fba71f --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message + +data class DeliveryMetadata( + val consumerTag: String, + val isRedelivered: Boolean +) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt index 81d1d01bc..ff9f0ce2e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt @@ -20,17 +20,14 @@ import com.rabbitmq.client.Delivery import java.io.IOException fun interface ManualAckDeliveryCallback { - /** - * Called when a delivery from queue is received - * @param consumerTag the _consumer_ tag associated with the consumer - * @param delivery the delivered message - * @param confirmProcessed the action that should be invoked when the message can be considered as processed - */ + @Throws(IOException::class) - fun handle(consumerTag: String, delivery: Delivery, confirmProcessed: Confirmation) + fun handle(deliveryMetadata: DeliveryMetadata, delivery: Delivery, confirmProcessed: Confirmation) - fun interface Confirmation { + interface Confirmation { @Throws(IOException::class) fun confirm() + @Throws(IOException::class) + fun reject() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt index 7603a39e7..de1ed91ef 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt @@ -17,8 +17,8 @@ package com.exactpro.th2.common.schema.message.impl import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback -import mu.KotlinLogging import java.util.concurrent.atomic.AtomicBoolean +import mu.KotlinLogging class OnlyOnceConfirmation private constructor( private val id: String, @@ -34,6 +34,14 @@ class OnlyOnceConfirmation private constructor( } } + override fun reject() { + if (called.compareAndSet(false, true)) { + delegate.reject() + } else { + LOGGER.warn { "Confirmation '$id' invoked more that one time" } + } + } + companion object { private val LOGGER = KotlinLogging.logger { } From 4330d6edd32119c916812ea01b0748539e0fa3a4 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 20 Oct 2022 11:23:54 +0400 Subject: [PATCH 063/154] [TH2-4299] merge fixes, readme update --- README.md | 1 + .../notification/NotificationEventBatchSubscriber.kt | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 945a94f40..3cc101645 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,7 @@ dependencies { + Removed `cradleInstanceName` parameter from `cradle.json` + Added `prepareStorage` property to `cradle.json` + `com.exactpro.th2.common.event.Event.toProto...()` by `parentEventId`/`bookName`/`(bookName + scope)` ++ Added `isRedelivered` flag to message ### 3.40.0 diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt index dff106246..8742f9a24 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt @@ -18,6 +18,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification import com.exactpro.th2.common.grpc.EventBatch import com.exactpro.th2.common.schema.message.ConfirmationMessageListener +import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.FilterFunction import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback import com.exactpro.th2.common.schema.message.MessageSubscriber @@ -25,8 +26,8 @@ import com.exactpro.th2.common.schema.message.SubscriberMonitor import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager import com.rabbitmq.client.Delivery -import mu.KotlinLogging import java.util.concurrent.CopyOnWriteArrayList +import mu.KotlinLogging class NotificationEventBatchSubscriber( private val connectionManager: ConnectionManager, @@ -58,11 +59,11 @@ class NotificationEventBatchSubscriber( override fun start() { monitor = connectionManager.basicConsume( queue, - { consumerTag: String, delivery: Delivery, confirmation: ManualAckDeliveryCallback.Confirmation -> + { deliveryMetadata: DeliveryMetadata, delivery: Delivery, confirmation: ManualAckDeliveryCallback.Confirmation -> try { for (listener in listeners) { try { - listener.handle(consumerTag, EventBatch.parseFrom(delivery.body), confirmation) + listener.handle(deliveryMetadata, EventBatch.parseFrom(delivery.body), confirmation) } catch (listenerExc: Exception) { LOGGER.warn( "Message listener from class '{}' threw exception", From 7d388be857ccd8ffd3db750285feec2c55f82833 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 20 Oct 2022 13:11:10 +0400 Subject: [PATCH 064/154] [TH2-4299] fixes according to MR comments --- .../message/impl/rabbitmq/AbstractRabbitSubscriber.java | 5 +++-- .../exactpro/th2/common/schema/message/DeliveryMetadata.kt | 2 +- .../th2/common/schema/message/ManualAckDeliveryCallback.kt | 6 ++++++ .../th2/common/schema/message/impl/OnlyOnceConfirmation.kt | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index dd1284cd3..d6a220eb8 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -129,6 +129,7 @@ public void start() throws Exception { try { value = valueFromBytes(delivery.getBody()); } catch (Exception e) { + LOGGER.error("Couldn't parse delivery. Reject message received", e); confirmProcessed.reject(); throw new IOException( String.format( @@ -238,11 +239,11 @@ protected void handle(String consumeTag, Delivery delivery, T value, Confirmatio confirmation.confirm(); } } catch (Exception e) { - LOGGER.error("Can not parse value from delivery for: {}", consumeTag, e); + LOGGER.error("Can not parse value from delivery for: {}. Reject message received", consumeTag, e); try { confirmation.reject(); } catch (IOException ex) { - LOGGER.error("Cannot confirm delivery for {}", consumeTag, ex); + LOGGER.error("Cannot reject delivery for {}", consumeTag, ex); } } } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt index 980fba71f..64319051a 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt index ff9f0ce2e..f9d6530b7 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ManualAckDeliveryCallback.kt @@ -21,6 +21,12 @@ import java.io.IOException fun interface ManualAckDeliveryCallback { + /** + * Called when a delivery from queue is received + * @param deliveryMetadata contains the _consumer_ tag associated with the consumer and _isRedelivered_ flag. + * @param delivery the delivered message + * @param confirmProcessed the action that should be invoked when the message can be considered as processed + */ @Throws(IOException::class) fun handle(deliveryMetadata: DeliveryMetadata, delivery: Delivery, confirmProcessed: Confirmation) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt index de1ed91ef..c76daf479 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/OnlyOnceConfirmation.kt @@ -30,7 +30,7 @@ class OnlyOnceConfirmation private constructor( if (called.compareAndSet(false, true)) { delegate.confirm() } else { - LOGGER.warn { "Confirmation '$id' invoked more that one time" } + LOGGER.warn { "Confirmation or rejection '$id' invoked more that one time" } } } @@ -38,7 +38,7 @@ class OnlyOnceConfirmation private constructor( if (called.compareAndSet(false, true)) { delegate.reject() } else { - LOGGER.warn { "Confirmation '$id' invoked more that one time" } + LOGGER.warn { "Confirmation or rejection '$id' invoked more that one time" } } } From c2bf1bf93914601967a2a08bc599d733ffa1331c Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 20 Oct 2022 14:36:35 +0400 Subject: [PATCH 065/154] [TH2-4299] removed try-catch from catch --- .../common/schema/event/EventBatchSubscriber.java | 12 ++++++------ .../impl/rabbitmq/AbstractRabbitSubscriber.java | 8 ++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java index b99619e54..f9010520b 100644 --- a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java @@ -18,19 +18,19 @@ import com.exactpro.th2.common.grpc.EventBatch; import com.exactpro.th2.common.schema.message.FilterFunction; +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; -import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.rabbitmq.client.Delivery; - -import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_PIN_LABEL; -import static com.exactpro.th2.common.schema.event.EventBatchRouter.EVENT_TYPE; import io.prometheus.client.Counter; - import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.io.IOException; + import static com.exactpro.th2.common.message.MessageUtils.toJson; +import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_PIN_LABEL; +import static com.exactpro.th2.common.schema.event.EventBatchRouter.EVENT_TYPE; public class EventBatchSubscriber extends AbstractRabbitSubscriber { private static final Counter EVENT_SUBSCRIBE_TOTAL = Counter.build() @@ -71,7 +71,7 @@ protected EventBatch filter(EventBatch eventBatch) throws Exception { @Override protected void handle(String consumeTag, Delivery delivery, EventBatch value, - Confirmation confirmation) { + Confirmation confirmation) throws IOException { EVENT_SUBSCRIBE_TOTAL .labels(th2Pin) .inc(value.getEventsCount()); diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index d6a220eb8..53671d781 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -204,7 +204,7 @@ protected boolean callFilterFunction(Message message, List "Received value from " + routingKey + " is null"); @@ -240,11 +240,7 @@ protected void handle(String consumeTag, Delivery delivery, T value, Confirmatio } } catch (Exception e) { LOGGER.error("Can not parse value from delivery for: {}. Reject message received", consumeTag, e); - try { - confirmation.reject(); - } catch (IOException ex) { - LOGGER.error("Cannot reject delivery for {}", consumeTag, ex); - } + confirmation.reject(); } } From e32f08f07444da097f881d400da4cdf3959d936b Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Fri, 21 Oct 2022 12:02:58 +0400 Subject: [PATCH 066/154] [TH2-4299] made fun interfaces out of ManualConfirmationListener and ConfirmationMessageListener --- .../th2/common/schema/message/ConfirmationMessageListener.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt index 2c31cbb3a..c8e8eac5d 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt @@ -16,7 +16,7 @@ package com.exactpro.th2.common.schema.message -interface ConfirmationMessageListener { +fun interface ConfirmationMessageListener { @Throws(Exception::class) @Deprecated( @@ -47,7 +47,7 @@ interface ConfirmationMessageListener { /** * The interface marker that indicates that acknowledge will be manually invoked by the listener itself */ -interface ManualConfirmationListener : ConfirmationMessageListener { +fun interface ManualConfirmationListener : ConfirmationMessageListener { /** * The listener must invoke the [confirmation] callback once it has processed the [message] * @see ConfirmationMessageListener.handle From b24a39afa0b827946880a79d16dae748b94e6689 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 26 Oct 2022 18:24:05 +0400 Subject: [PATCH 067/154] [TH2-4262] disabled incrementTotalMetrics function --- src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt b/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt index ead708ff0..1c77579be 100644 --- a/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt @@ -30,6 +30,8 @@ fun incrementTotalMetrics( groupCounter: Counter, gauge: Gauge ) { + if (true) return //FIXME: added for test + val groupsBySessionAliasAndDirection = mutableMapOf() batch.groupsList.forEach { group -> if (group.messagesList.isNotEmpty()) { From 804f470423d1a63f151b0e2dee55e4338f93fde5 Mon Sep 17 00:00:00 2001 From: georgiano Date: Thu, 27 Oct 2022 14:37:07 +0400 Subject: [PATCH 068/154] [TH2-4299] subscribeAllWithManualAck now takes ManualConfirmationListener --- .../common/schema/message/MessageRouter.java | 6 ++-- .../message/ConfirmationMessageListener.kt | 30 ++++++++++++++++++- .../impl/rabbitmq/AbstractRabbitRouter.kt | 17 ++++------- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index a21672179..9ba48bd80 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -87,7 +87,7 @@ default SubscriberMonitor subscribeAll(MessageListener callback) { * @throws IllegalStateException when more than 1 queue is found * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue */ - default SubscriberMonitor subscribeWithManualAck(ConfirmationMessageListener callback, String... queueAttr) { + default SubscriberMonitor subscribeWithManualAck(ManualConfirmationListener callback, String... queueAttr) { // TODO: probably should not have default implementation throw new UnsupportedOperationException("The subscription with manual confirmation is not supported"); } @@ -97,7 +97,7 @@ default SubscriberMonitor subscribeWithManualAck(ConfirmationMessageListener * @param callback listener with manual confirmation * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue */ - default SubscriberMonitor subscribeAllWithManualAck(ConfirmationMessageListener callback) { + default SubscriberMonitor subscribeAllWithManualAck(ManualConfirmationListener callback) { // TODO: probably should not have default implementation return subscribeAllWithManualAck(callback, QueueAttribute.SUBSCRIBE.toString()); } @@ -108,7 +108,7 @@ default SubscriberMonitor subscribeAllWithManualAck(ConfirmationMessageListener< * @param queueAttr queues attributes * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue */ - default SubscriberMonitor subscribeAllWithManualAck(ConfirmationMessageListener callback, String... queueAttr) { + default SubscriberMonitor subscribeAllWithManualAck(ManualConfirmationListener callback, String... queueAttr) { // TODO: probably should not have default implementation throw new UnsupportedOperationException("The subscription with manual confirmation is not supported"); } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt index c8e8eac5d..9749f7a89 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt @@ -52,7 +52,20 @@ fun interface ManualConfirmationListener : ConfirmationMessageListener { * The listener must invoke the [confirmation] callback once it has processed the [message] * @see ConfirmationMessageListener.handle */ - override fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) + @Deprecated( + "This method does not provide all necessary information about a message", + ReplaceWith("handle(deliveryMetadata, message, confirmation)") + ) + override fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { + } + + @Throws(Exception::class) + override fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) + + companion object { + @JvmStatic + fun wrap(listener: MessageListener): ManualConfirmationListener = ManualDelegateListener(listener) + } } private class DelegateListener( @@ -67,5 +80,20 @@ private class DelegateListener( delegate.onClose() } + override fun toString(): String = "Delegate($delegate)" +} + +private class ManualDelegateListener( + private val delegate: MessageListener, +) : ManualConfirmationListener { + + override fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { + delegate.handle(deliveryMetadata, message) + } + + override fun onClose() { + delegate.onClose() + } + override fun toString(): String = "Delegate($delegate)" } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index 6f3c66730..b20691480 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -17,16 +17,9 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.exception.RouterException import com.exactpro.th2.common.schema.filter.strategy.FilterStrategy -import com.exactpro.th2.common.schema.message.ConfirmationMessageListener -import com.exactpro.th2.common.schema.message.MessageListener -import com.exactpro.th2.common.schema.message.MessageRouter -import com.exactpro.th2.common.schema.message.MessageRouterContext -import com.exactpro.th2.common.schema.message.MessageSender -import com.exactpro.th2.common.schema.message.MessageSubscriber +import com.exactpro.th2.common.schema.message.* import com.exactpro.th2.common.schema.message.QueueAttribute.PUBLISH import com.exactpro.th2.common.schema.message.QueueAttribute.SUBSCRIBE -import com.exactpro.th2.common.schema.message.SubscriberMonitor -import com.exactpro.th2.common.schema.message.appendAttributes import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter @@ -95,14 +88,14 @@ abstract class AbstractRabbitRouter : MessageRouter { } override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor { - return subscribeWithManualAck(ConfirmationMessageListener.wrap(callback), *attributes) + return subscribeWithManualAck(ManualConfirmationListener.wrap(callback), *attributes) } override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor { - return subscribeAllWithManualAck(ConfirmationMessageListener.wrap(callback), *attributes) + return subscribeAllWithManualAck(ManualConfirmationListener.wrap(callback), *attributes) } - override fun subscribeWithManualAck(callback: ConfirmationMessageListener, vararg attributes: String): SubscriberMonitor { + override fun subscribeWithManualAck(callback: ManualConfirmationListener, vararg attributes: String): SubscriberMonitor { val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } return subscribe(pintAttributes, callback) { check(size == 1) { @@ -111,7 +104,7 @@ abstract class AbstractRabbitRouter : MessageRouter { } } - override fun subscribeAllWithManualAck(callback: ConfirmationMessageListener, vararg attributes: String): SubscriberMonitor { + override fun subscribeAllWithManualAck(callback: ManualConfirmationListener, vararg attributes: String): SubscriberMonitor { val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } return subscribe(pintAttributes, callback) { check(isNotEmpty()) { From 2bc3a95803c863c86f93675a974dd5830362906f Mon Sep 17 00:00:00 2001 From: georgiano Date: Thu, 27 Oct 2022 15:23:27 +0400 Subject: [PATCH 069/154] [TH2-4299] added kotlin compiler flag to compile java compatible interface --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index fa5581c61..24703f9d2 100644 --- a/build.gradle +++ b/build.gradle @@ -244,6 +244,7 @@ sourceSets { tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { kotlinOptions.jvmTarget = "11" + kotlinOptions.freeCompilerArgs += "-Xjvm-default=all" } clean { From 81d1403d20ce0e833f3bc58921b6df64a9e44063 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Mon, 31 Oct 2022 13:08:03 +0400 Subject: [PATCH 070/154] [TH2-4299] default inner use of metadata handle --- .../th2/common/schema/message/ConfirmationMessageListener.kt | 2 +- .../exactpro/th2/common/schema/message/DeliveryMetadata.kt | 2 +- .../message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt index 9749f7a89..44a5d639f 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt @@ -73,7 +73,7 @@ private class DelegateListener( ) : ConfirmationMessageListener { override fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { - delegate.handle(consumerTag, message) + delegate.handle(DeliveryMetadata(consumerTag), message) } override fun onClose() { diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt index 64319051a..e844452b3 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt @@ -17,5 +17,5 @@ package com.exactpro.th2.common.schema.message data class DeliveryMetadata( val consumerTag: String, - val isRedelivered: Boolean + val isRedelivered: Boolean = false ) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index 76c203e43..fe72bb113 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -16,6 +16,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq import com.exactpro.th2.common.grpc.MessageGroupBatch +import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.MessageRouterContext @@ -41,7 +42,7 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { return groupBatchRouter.subscribe({ consumerTag: String, message: MessageGroupBatch -> - callback.handle(consumerTag, buildFromGroupBatch(message)) + callback.handle(DeliveryMetadata(consumerTag), buildFromGroupBatch(message)) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) @@ -49,7 +50,7 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { return groupBatchRouter.subscribeAll({ consumerTag: String, message: MessageGroupBatch -> - callback.handle(consumerTag, buildFromGroupBatch(message)) + callback.handle(DeliveryMetadata(consumerTag), buildFromGroupBatch(message)) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) From 91baefca8954a3cd6f41b0874b4586fb18a0b55d Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 31 Oct 2022 17:12:29 +0400 Subject: [PATCH 071/154] [TH2-4299] tmp --- .../schema/message/MessageListener.java | 2 - .../common/schema/message/MessageRouter.java | 23 ++++++++ .../schema/message/MessageSubscriber.java | 2 +- .../rabbitmq/AbstractRabbitSubscriber.java | 11 ++-- .../message/ConfirmationMessageListener.kt | 55 ++++++++++++++++--- .../impl/rabbitmq/AbstractRabbitRouter.kt | 52 ++++++++++++++++-- 6 files changed, 122 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java index 4f312e527..f7f1713a6 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java @@ -31,6 +31,4 @@ default void handle(DeliveryMetadata deliveryMetadata, T message) throws Excepti } default void onClose() {} - - } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index 9ba48bd80..9d041778f 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -61,8 +61,18 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< * @throws IllegalStateException when more than 1 queue is found * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue */ + @Deprecated SubscriberMonitor subscribe(MessageListener callback, String... queueAttr); + /** + * Listen ONE RabbitMQ queue by intersection schemas queues attributes + * @param callback listener + * @param queueAttr queues attributes + * @throws IllegalStateException when more than 1 queue is found + * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + */ + SubscriberMonitor subscribe(AutoConfirmationListener callback, String... queueAttr); + /** * Listen ALL RabbitMQ queues in configurations * @param callback listener @@ -80,6 +90,19 @@ default SubscriberMonitor subscribeAll(MessageListener callback) { */ SubscriberMonitor subscribeAll(MessageListener callback, String... queueAttr); + /** + * Listen ONE RabbitMQ queue by intersection schemas queues attributes + * @param queueAttr queues attributes + * @param callback listener with manual confirmation + * @throws IllegalStateException when more than 1 queue is found + * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + */ + @Deprecated + default SubscriberMonitor subscribeWithManualAck(ConfirmationMessageListener callback, String... queueAttr) { + // TODO: probably should not have default implementation + throw new UnsupportedOperationException("The subscription with manual confirmation is not supported"); + } + /** * Listen ONE RabbitMQ queue by intersection schemas queues attributes * @param queueAttr queues attributes diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java index 0bc1ba7da..40be43425 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java @@ -37,5 +37,5 @@ public interface MessageSubscriber extends AutoCloseable { void start() throws Exception; - void addListener(ConfirmationMessageListener messageListener); + void addListener(ConfirmationListener messageListener); } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index 53671d781..f7c531c42 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -16,6 +16,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq; import com.exactpro.th2.common.metrics.HealthMetrics; +import com.exactpro.th2.common.schema.message.ConfirmationListener; import com.exactpro.th2.common.schema.message.ConfirmationMessageListener; import com.exactpro.th2.common.schema.message.DeliveryMetadata; import com.exactpro.th2.common.schema.message.FilterFunction; @@ -68,7 +69,7 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber> listeners = new CopyOnWriteArrayList<>(); + private final List> listeners = new CopyOnWriteArrayList<>(); private final String queue; private final AtomicReference connectionManager = new AtomicReference<>(); private final AtomicReference consumerMonitor = new AtomicReference<>(); @@ -160,8 +161,8 @@ public void start() throws Exception { } @Override - public void addListener(ConfirmationMessageListener messageListener) { - if (ConfirmationMessageListener.isManual(messageListener)) { + public void addListener(ConfirmationListener messageListener) { + if (ConfirmationListener.isManual(messageListener)) { if (!hasManualSubscriber.compareAndSet(false, true)) { throw new IllegalStateException("cannot subscribe listener " + messageListener + " because only one listener with manual confirmation is allowed per queue"); @@ -182,7 +183,7 @@ public void close() throws Exception { monitor.unsubscribe(); } - listeners.forEach(ConfirmationMessageListener::onClose); + listeners.forEach(ConfirmationListener::onClose); listeners.clear(); } @@ -224,7 +225,7 @@ protected void handle(String consumeTag, Delivery delivery, T value, Confirmatio } boolean hasManualConfirmation = false; - for (ConfirmationMessageListener listener : listeners) { + for (ConfirmationListener listener : listeners) { try { boolean redeliver = delivery.getEnvelope().isRedeliver(); listener.handle(new DeliveryMetadata(consumeTag, redeliver), filteredValue, confirmation); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt index 44a5d639f..d0ddfef58 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt @@ -16,7 +16,8 @@ package com.exactpro.th2.common.schema.message -fun interface ConfirmationMessageListener { +@Deprecated("") //FIXME: write message +interface ConfirmationMessageListener: ConfirmationListener { @Throws(Exception::class) @Deprecated( @@ -26,28 +27,59 @@ fun interface ConfirmationMessageListener { fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) @Throws(Exception::class) - fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { + override fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { handle(deliveryMetadata.consumerTag, message, confirmation) } + companion object { + @JvmStatic + fun wrap(listener: MessageListener): ConfirmationListener = DelegateListener(listener) + + @JvmStatic + fun wrap(listener: AutoConfirmationListener): ConfirmationListener = DelegateListener(listener) + + /** + * @return `true` if the listener uses manual acknowledgment + */ + @JvmStatic + fun isManual(listener: ConfirmationListener<*>): Boolean = listener is ManualConfirmationListener<*> + } +} + +fun interface AutoConfirmationListener : MessageListener { + + override fun handle(consumerTag: String?, message: T) { + throw UnsupportedOperationException() + } + + override fun handle(deliveryMetadata: DeliveryMetadata, message: T) +} + +interface ConfirmationListener { + @Throws(Exception::class) + fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) + fun onClose() {} companion object { @JvmStatic - fun wrap(listener: MessageListener): ConfirmationMessageListener = DelegateListener(listener) + fun wrap(listener: MessageListener): ConfirmationListener = DelegateListener(listener) + + @JvmStatic + fun wrap(listener: AutoConfirmationListener): ConfirmationListener = DelegateListener(listener) /** * @return `true` if the listener uses manual acknowledgment */ @JvmStatic - fun isManual(listener: ConfirmationMessageListener<*>): Boolean = listener is ManualConfirmationListener<*> + fun isManual(listener: ConfirmationListener<*>): Boolean = listener is ManualConfirmationListener<*> } } /** * The interface marker that indicates that acknowledge will be manually invoked by the listener itself */ -fun interface ManualConfirmationListener : ConfirmationMessageListener { +fun interface ManualConfirmationListener : ConfirmationListener { /** * The listener must invoke the [confirmation] callback once it has processed the [message] * @see ConfirmationMessageListener.handle @@ -56,7 +88,8 @@ fun interface ManualConfirmationListener : ConfirmationMessageListener { "This method does not provide all necessary information about a message", ReplaceWith("handle(deliveryMetadata, message, confirmation)") ) - override fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { + fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { + throw UnsupportedOperationException() } @Throws(Exception::class) @@ -70,10 +103,14 @@ fun interface ManualConfirmationListener : ConfirmationMessageListener { private class DelegateListener( private val delegate: MessageListener, -) : ConfirmationMessageListener { +) : ConfirmationListener { - override fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { - delegate.handle(DeliveryMetadata(consumerTag), message) + override fun handle( + deliveryMetadata: DeliveryMetadata, + message: T, + confirmation: ManualAckDeliveryCallback.Confirmation + ) { + delegate.handle(deliveryMetadata, message) } override fun onClose() { diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index b20691480..59ee4e566 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -87,19 +87,49 @@ abstract class AbstractRabbitRouter : MessageRouter { } } + @Deprecated("") //FIXME: override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor { - return subscribeWithManualAck(ManualConfirmationListener.wrap(callback), *attributes) + return subscribe(attributes = attributes, ConfirmationListener.wrap(callback)) { + check(size == 1) { + "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $attributes and filters, expected 1, actual $size" + } + } + } + + override fun subscribe(callback: AutoConfirmationListener, vararg queueAttr: String): SubscriberMonitor { + return subscribe(attributes = queueAttr, ConfirmationListener.wrap(callback)) { + check(size == 1) { + "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $queueAttr and filters, expected 1, actual $size" + } + } } override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor { - return subscribeAllWithManualAck(ManualConfirmationListener.wrap(callback), *attributes) + return subscribeAllWithManualAck(ManualConfirmationListener.wrap(callback), *attributes) //FIXME: + } + +// override fun subscribeWithManualAck( +// callback: ConfirmationMessageListener?, +// vararg queueAttr: String? +// ): SubscriberMonitor { +// return super.subscribeWithManualAck(callback, *queueAttr) +// } + + override fun subscribeWithManualAck( + callback: ConfirmationMessageListener, + vararg queueAttr: String + ): SubscriberMonitor { + return subscribe(attributes = queueAttr, callback) { + check(size == 1) { + "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $queueAttr and filters, expected 1, actual $size" + } + } } override fun subscribeWithManualAck(callback: ManualConfirmationListener, vararg attributes: String): SubscriberMonitor { - val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } - return subscribe(pintAttributes, callback) { + return subscribe(attributes = attributes, callback) { check(size == 1) { - "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $pintAttributes and filters, expected 1, actual $size" + "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $attributes and filters, expected 1, actual $size" } } } @@ -182,9 +212,19 @@ abstract class AbstractRabbitRouter : MessageRouter { checkOrThrow(exceptions.values) { "Can't send to pin(s): ${exceptions.keys}" } } + private fun subscribe( + vararg attributes: String, + callback: ConfirmationListener, + check: List.() -> Unit + ): SubscriberMonitor { + val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } + + return subscribe(pintAttributes, callback, check) + } + private fun subscribe( pintAttributes: Set, - messageListener: ConfirmationMessageListener, + messageListener: ConfirmationListener, check: List.() -> Unit ): SubscriberMonitor { val packages: List = configuration.queues.asSequence() From efee967d703628dda5808e2639389c0dba5de010 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 3 Nov 2022 12:43:56 +0400 Subject: [PATCH 072/154] [TH2-4299] removed deprecated listeners --- .../schema/message/MessageListener.java | 9 +-- .../common/schema/message/MessageRouter.java | 43 ----------- .../rabbitmq/AbstractRabbitSubscriber.java | 3 +- .../message/ConfirmationMessageListener.kt | 77 +------------------ .../schema/message/MessageRouterUtils.kt | 6 ++ .../AbstractGroupBatchAdapterRouter.kt | 14 ++-- .../impl/rabbitmq/AbstractRabbitRouter.kt | 56 ++++++-------- .../NotificationEventBatchRouter.kt | 4 +- .../NotificationEventBatchSubscriber.kt | 8 +- .../TestConfirmationMessageListenerWrapper.kt | 8 +- .../TestRabbitMessageGroupBatchRouter.kt | 7 ++ 11 files changed, 62 insertions(+), 173 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java index f7f1713a6..b0912165d 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageListener.java @@ -21,14 +21,7 @@ */ public interface MessageListener { - @Deprecated( - since = "This method does not provide all necessary information about a message" - ) - void handle(String consumerTag, T message) throws Exception; - - default void handle(DeliveryMetadata deliveryMetadata, T message) throws Exception { - handle(deliveryMetadata.getConsumerTag(), message); - } + void handle(DeliveryMetadata deliveryMetadata, T message) throws Exception; default void onClose() {} } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index 9d041778f..25c0a8e1a 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -20,7 +20,6 @@ import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration; import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; - import org.jetbrains.annotations.NotNull; import java.io.IOException; @@ -61,27 +60,8 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< * @throws IllegalStateException when more than 1 queue is found * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue */ - @Deprecated SubscriberMonitor subscribe(MessageListener callback, String... queueAttr); - /** - * Listen ONE RabbitMQ queue by intersection schemas queues attributes - * @param callback listener - * @param queueAttr queues attributes - * @throws IllegalStateException when more than 1 queue is found - * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue - */ - SubscriberMonitor subscribe(AutoConfirmationListener callback, String... queueAttr); - - /** - * Listen ALL RabbitMQ queues in configurations - * @param callback listener - * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue - */ - default SubscriberMonitor subscribeAll(MessageListener callback) { - return subscribeAll(callback, QueueAttribute.SUBSCRIBE.toString()); - } - /** * Listen SOME RabbitMQ queues by intersection schemas queues attributes * @param callback listener @@ -90,19 +70,6 @@ default SubscriberMonitor subscribeAll(MessageListener callback) { */ SubscriberMonitor subscribeAll(MessageListener callback, String... queueAttr); - /** - * Listen ONE RabbitMQ queue by intersection schemas queues attributes - * @param queueAttr queues attributes - * @param callback listener with manual confirmation - * @throws IllegalStateException when more than 1 queue is found - * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue - */ - @Deprecated - default SubscriberMonitor subscribeWithManualAck(ConfirmationMessageListener callback, String... queueAttr) { - // TODO: probably should not have default implementation - throw new UnsupportedOperationException("The subscription with manual confirmation is not supported"); - } - /** * Listen ONE RabbitMQ queue by intersection schemas queues attributes * @param queueAttr queues attributes @@ -115,16 +82,6 @@ default SubscriberMonitor subscribeWithManualAck(ManualConfirmationListener c throw new UnsupportedOperationException("The subscription with manual confirmation is not supported"); } - /** - * Listen ALL RabbitMQ queues in configurations - * @param callback listener with manual confirmation - * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue - */ - default SubscriberMonitor subscribeAllWithManualAck(ManualConfirmationListener callback) { - // TODO: probably should not have default implementation - return subscribeAllWithManualAck(callback, QueueAttribute.SUBSCRIBE.toString()); - } - /** * Listen SOME RabbitMQ queues by intersection schemas queues attributes * @param callback listener with manual confirmation diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index f7c531c42..e2bbbc542 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -17,7 +17,6 @@ import com.exactpro.th2.common.metrics.HealthMetrics; import com.exactpro.th2.common.schema.message.ConfirmationListener; -import com.exactpro.th2.common.schema.message.ConfirmationMessageListener; import com.exactpro.th2.common.schema.message.DeliveryMetadata; import com.exactpro.th2.common.schema.message.FilterFunction; import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; @@ -230,7 +229,7 @@ protected void handle(String consumeTag, Delivery delivery, T value, Confirmatio boolean redeliver = delivery.getEnvelope().isRedeliver(); listener.handle(new DeliveryMetadata(consumeTag, redeliver), filteredValue, confirmation); if (!hasManualConfirmation) { - hasManualConfirmation = ConfirmationMessageListener.isManual(listener); + hasManualConfirmation = ConfirmationListener.isManual(listener); } } catch (Exception listenerExc) { LOGGER.warn("Message listener from class '{}' threw exception", listener.getClass(), listenerExc); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt index d0ddfef58..50a078adb 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ConfirmationMessageListener.kt @@ -16,45 +16,6 @@ package com.exactpro.th2.common.schema.message -@Deprecated("") //FIXME: write message -interface ConfirmationMessageListener: ConfirmationListener { - - @Throws(Exception::class) - @Deprecated( - "This method does not provide all necessary information about a message", - ReplaceWith("handle(deliveryMetadata, message, confirmation)") - ) - fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) - - @Throws(Exception::class) - override fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { - handle(deliveryMetadata.consumerTag, message, confirmation) - } - - companion object { - @JvmStatic - fun wrap(listener: MessageListener): ConfirmationListener = DelegateListener(listener) - - @JvmStatic - fun wrap(listener: AutoConfirmationListener): ConfirmationListener = DelegateListener(listener) - - /** - * @return `true` if the listener uses manual acknowledgment - */ - @JvmStatic - fun isManual(listener: ConfirmationListener<*>): Boolean = listener is ManualConfirmationListener<*> - } -} - -fun interface AutoConfirmationListener : MessageListener { - - override fun handle(consumerTag: String?, message: T) { - throw UnsupportedOperationException() - } - - override fun handle(deliveryMetadata: DeliveryMetadata, message: T) -} - interface ConfirmationListener { @Throws(Exception::class) fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) @@ -63,10 +24,7 @@ interface ConfirmationListener { companion object { @JvmStatic - fun wrap(listener: MessageListener): ConfirmationListener = DelegateListener(listener) - - @JvmStatic - fun wrap(listener: AutoConfirmationListener): ConfirmationListener = DelegateListener(listener) + fun wrap(listener: MessageListener): ConfirmationListener = AutoConfirmationListener(listener) /** * @return `true` if the listener uses manual acknowledgment @@ -80,28 +38,16 @@ interface ConfirmationListener { * The interface marker that indicates that acknowledge will be manually invoked by the listener itself */ fun interface ManualConfirmationListener : ConfirmationListener { + /** * The listener must invoke the [confirmation] callback once it has processed the [message] - * @see ConfirmationMessageListener.handle + * @see ConfirmationListener.handle */ - @Deprecated( - "This method does not provide all necessary information about a message", - ReplaceWith("handle(deliveryMetadata, message, confirmation)") - ) - fun handle(consumerTag: String, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { - throw UnsupportedOperationException() - } - @Throws(Exception::class) override fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) - - companion object { - @JvmStatic - fun wrap(listener: MessageListener): ManualConfirmationListener = ManualDelegateListener(listener) - } } -private class DelegateListener( +private class AutoConfirmationListener( private val delegate: MessageListener, ) : ConfirmationListener { @@ -119,18 +65,3 @@ private class DelegateListener( override fun toString(): String = "Delegate($delegate)" } - -private class ManualDelegateListener( - private val delegate: MessageListener, -) : ManualConfirmationListener { - - override fun handle(deliveryMetadata: DeliveryMetadata, message: T, confirmation: ManualAckDeliveryCallback.Confirmation) { - delegate.handle(deliveryMetadata, message) - } - - override fun onClose() { - delegate.onClose() - } - - override fun toString(): String = "Delegate($delegate)" -} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index 8480e6e8e..49d39ce60 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -101,4 +101,10 @@ fun MessageGroupBatch.toShortDebugString(): String = buildString { .apply { append(this) } append(')') +} + +fun addSubscribeAttributeByDefault(vararg attributes: String): Array { + if (attributes.isNotEmpty()) + return attributes + return arrayOf(QueueAttribute.SUBSCRIBE.toString()) } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index fe72bb113..d46cfaac1 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -16,11 +16,13 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq import com.exactpro.th2.common.grpc.MessageGroupBatch +import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.MessageRouterContext import com.exactpro.th2.common.schema.message.SubscriberMonitor +import com.exactpro.th2.common.schema.message.addSubscribeAttributeByDefault import com.exactpro.th2.common.schema.message.appendAttributes abstract class AbstractGroupBatchAdapterRouter : MessageRouter { @@ -41,18 +43,20 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { } override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { - return groupBatchRouter.subscribe({ consumerTag: String, message: MessageGroupBatch -> - callback.handle(DeliveryMetadata(consumerTag), buildFromGroupBatch(message)) + val listener = ConfirmationListener.wrap(callback) + return groupBatchRouter.subscribeWithManualAck({ consumerTag: DeliveryMetadata, message: MessageGroupBatch, confirmation -> + listener.handle(consumerTag, buildFromGroupBatch(message), confirmation) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) } override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { - return groupBatchRouter.subscribeAll({ consumerTag: String, message: MessageGroupBatch -> - callback.handle(DeliveryMetadata(consumerTag), buildFromGroupBatch(message)) + val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) + return groupBatchRouter.subscribeAll({ deliveryMetadata: DeliveryMetadata, message: MessageGroupBatch -> + callback.handle(deliveryMetadata, buildFromGroupBatch(message)) }, - *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() + *appendAttributes(*attributesWithDefaultValue) { getRequiredSubscribeAttributes() }.toTypedArray() ) } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index 59ee4e566..d22c0e10e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -17,17 +17,26 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.exception.RouterException import com.exactpro.th2.common.schema.filter.strategy.FilterStrategy -import com.exactpro.th2.common.schema.message.* +import com.exactpro.th2.common.schema.message.ConfirmationListener +import com.exactpro.th2.common.schema.message.ManualConfirmationListener +import com.exactpro.th2.common.schema.message.MessageListener +import com.exactpro.th2.common.schema.message.MessageRouter +import com.exactpro.th2.common.schema.message.MessageRouterContext +import com.exactpro.th2.common.schema.message.MessageSender +import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.QueueAttribute.PUBLISH import com.exactpro.th2.common.schema.message.QueueAttribute.SUBSCRIBE +import com.exactpro.th2.common.schema.message.SubscriberMonitor +import com.exactpro.th2.common.schema.message.addSubscribeAttributeByDefault +import com.exactpro.th2.common.schema.message.appendAttributes import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager import com.google.protobuf.Message -import mu.KotlinLogging import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference +import mu.KotlinLogging typealias PinName = String typealias PinConfiguration = QueueConfiguration @@ -87,55 +96,38 @@ abstract class AbstractRabbitRouter : MessageRouter { } } - @Deprecated("") //FIXME: override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor { + val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } return subscribe(attributes = attributes, ConfirmationListener.wrap(callback)) { check(size == 1) { - "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $attributes and filters, expected 1, actual $size" - } - } - } - - override fun subscribe(callback: AutoConfirmationListener, vararg queueAttr: String): SubscriberMonitor { - return subscribe(attributes = queueAttr, ConfirmationListener.wrap(callback)) { - check(size == 1) { - "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $queueAttr and filters, expected 1, actual $size" + "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $pintAttributes and filters, expected 1, actual $size" } } } override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor { - return subscribeAllWithManualAck(ManualConfirmationListener.wrap(callback), *attributes) //FIXME: - } - -// override fun subscribeWithManualAck( -// callback: ConfirmationMessageListener?, -// vararg queueAttr: String? -// ): SubscriberMonitor { -// return super.subscribeWithManualAck(callback, *queueAttr) -// } - - override fun subscribeWithManualAck( - callback: ConfirmationMessageListener, - vararg queueAttr: String - ): SubscriberMonitor { - return subscribe(attributes = queueAttr, callback) { - check(size == 1) { - "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $queueAttr and filters, expected 1, actual $size" + val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) + val pintAttributes: Set = appendAttributes(*attributesWithDefaultValue) { getRequiredSubscribeAttributes() } + val listener = ConfirmationListener.wrap(callback) + return subscribe(pintAttributes = pintAttributes, listener) { + check(isNotEmpty()) { + "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe all operation by attributes $pintAttributes and filters, expected 1 or more, actual $size" } } } override fun subscribeWithManualAck(callback: ManualConfirmationListener, vararg attributes: String): SubscriberMonitor { - return subscribe(attributes = attributes, callback) { + val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } + return subscribe(pintAttributes, callback) { check(size == 1) { - "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $attributes and filters, expected 1, actual $size" + "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $pintAttributes and filters, expected 1, actual $size" } } } override fun subscribeAllWithManualAck(callback: ManualConfirmationListener, vararg attributes: String): SubscriberMonitor { - val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } + val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) + val pintAttributes: Set = appendAttributes(*attributesWithDefaultValue) { getRequiredSubscribeAttributes() } return subscribe(pintAttributes, callback) { check(isNotEmpty()) { "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe all operation by attributes $pintAttributes and filters, expected 1 or more, actual $size" diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt index b81b06642..7c3bb7c89 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchRouter.kt @@ -17,7 +17,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification import com.exactpro.th2.common.grpc.EventBatch import com.exactpro.th2.common.schema.exception.RouterException -import com.exactpro.th2.common.schema.message.ConfirmationMessageListener +import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouterContext import com.exactpro.th2.common.schema.message.NotificationRouter @@ -55,7 +55,7 @@ class NotificationEventBatchRouter : NotificationRouter { override fun subscribe(callback: MessageListener): SubscriberMonitor { try { - subscriber.addListener(ConfirmationMessageListener.wrap(callback)) + subscriber.addListener(ConfirmationListener.wrap(callback)) subscriber.start() } catch (e: Exception) { val errorMessage = "Listener can't be subscribed via the queue $queue" diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt index 8742f9a24..d09fc784e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt @@ -17,7 +17,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification import com.exactpro.th2.common.grpc.EventBatch -import com.exactpro.th2.common.schema.message.ConfirmationMessageListener +import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.FilterFunction import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback @@ -33,7 +33,7 @@ class NotificationEventBatchSubscriber( private val connectionManager: ConnectionManager, private val queue: String ) : MessageSubscriber { - private val listeners = CopyOnWriteArrayList>() + private val listeners = CopyOnWriteArrayList>() private lateinit var monitor: SubscriberMonitor @Deprecated( @@ -80,13 +80,13 @@ class NotificationEventBatchSubscriber( ) } - override fun addListener(messageListener: ConfirmationMessageListener) { + override fun addListener(messageListener: ConfirmationListener) { listeners.add(messageListener) } override fun close() { monitor.unsubscribe() - listeners.forEach(ConfirmationMessageListener::onClose) + listeners.forEach(ConfirmationListener::onClose) listeners.clear() } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt index 0f6cd08e3..8d1e2ad80 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt @@ -26,20 +26,20 @@ import org.mockito.kotlin.verify class TestConfirmationMessageListenerWrapper { @Test fun `calls confirmation when requested`() { - val listener = ConfirmationMessageListener.wrap { _, _ -> } + val listener = ConfirmationListener.wrap { _, _ -> } mock {}.also { - listener.handle("", 2, it) + listener.handle(DeliveryMetadata(""), 2, it) verify(it, never()).confirm() } } @Test fun `calls confirmation when requested and method throw an exception`() { - val listener = ConfirmationMessageListener.wrap { _, _ -> error("test") } + val listener = ConfirmationListener.wrap { _, _ -> error("test") } mock {}.also { - assertThrows(IllegalStateException::class.java) { listener.handle("", 2, it) }.apply { + assertThrows(IllegalStateException::class.java) { listener.handle(DeliveryMetadata("test"), 2, it) }.apply { assertEquals("test", message) } verify(it, never()).confirm() diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt index 6010dd274..4acb14652 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt @@ -220,6 +220,13 @@ class TestRabbitMessageGroupBatchRouter { verify(connectionManager).basicConsume(eq("queue1"), any(), any()) } + @Test + fun `subscribes with manual ack to correct queue`() { + val monitor = router.subscribeWithManualAck(mock { }, "1") + Assertions.assertNotNull(monitor) { "monitor must not be null" } + + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + } @Test fun `subscribes to all matched queues`() { From c8a52378eb75edc2fd846670df518923b25ee383 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 3 Nov 2022 13:30:36 +0400 Subject: [PATCH 073/154] [TH2-4299] added array constant --- .../th2/common/schema/message/MessageRouterUtils.kt | 3 ++- .../impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index 49d39ce60..529db313e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -103,8 +103,9 @@ fun MessageGroupBatch.toShortDebugString(): String = buildString { append(')') } +private val subscribeArray = arrayOf(QueueAttribute.SUBSCRIBE.toString()) fun addSubscribeAttributeByDefault(vararg attributes: String): Array { if (attributes.isNotEmpty()) return attributes - return arrayOf(QueueAttribute.SUBSCRIBE.toString()) + return subscribeArray } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index d46cfaac1..853e44559 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -16,7 +16,6 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq import com.exactpro.th2.common.grpc.MessageGroupBatch -import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouter @@ -43,9 +42,8 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { } override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { - val listener = ConfirmationListener.wrap(callback) - return groupBatchRouter.subscribeWithManualAck({ consumerTag: DeliveryMetadata, message: MessageGroupBatch, confirmation -> - listener.handle(consumerTag, buildFromGroupBatch(message), confirmation) + return groupBatchRouter.subscribeWithManualAck({ deliveryMetadata: DeliveryMetadata, message: MessageGroupBatch, _ -> + callback.handle(deliveryMetadata, buildFromGroupBatch(message)) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) @@ -61,16 +59,18 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { } override fun send(messageBatch: T, vararg attributes: String) { + val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) groupBatchRouter.send( buildGroupBatch(messageBatch), - *appendAttributes(*attributes) { getRequiredSendAttributes() }.toTypedArray() + *appendAttributes(*attributesWithDefaultValue) { getRequiredSendAttributes() }.toTypedArray() ) } override fun sendAll(messageBatch: T, vararg attributes: String) { + val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) groupBatchRouter.sendAll( buildGroupBatch(messageBatch), - *appendAttributes(*attributes) { getRequiredSendAttributes() }.toTypedArray() + *appendAttributes(*attributesWithDefaultValue) { getRequiredSendAttributes() }.toTypedArray() ) } From 997b952322c323f87a69f5b597e5f9f6912317c8 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 3 Nov 2022 15:15:22 +0400 Subject: [PATCH 074/154] [TH2-4299] changed default constant for send, using another subscribe method --- .../th2/common/schema/message/MessageRouterUtils.kt | 12 ++++++++++-- .../impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt | 11 +++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index 529db313e..c7ad4f69a 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -30,6 +30,7 @@ import com.exactpro.th2.common.message.logId import com.exactpro.th2.common.message.toJson import com.exactpro.th2.common.schema.message.QueueAttribute.EVENT import com.exactpro.th2.common.schema.message.QueueAttribute.PUBLISH +import com.exactpro.th2.common.schema.message.QueueAttribute.SUBSCRIBE import com.google.protobuf.MessageOrBuilder import org.apache.commons.lang3.exception.ExceptionUtils @@ -103,9 +104,16 @@ fun MessageGroupBatch.toShortDebugString(): String = buildString { append(')') } -private val subscribeArray = arrayOf(QueueAttribute.SUBSCRIBE.toString()) +private val SUBSCRIBE_DEFAULT_ATTRIBUTES = arrayOf(SUBSCRIBE.toString()) fun addSubscribeAttributeByDefault(vararg attributes: String): Array { if (attributes.isNotEmpty()) return attributes - return subscribeArray + return SUBSCRIBE_DEFAULT_ATTRIBUTES +} + +private val PUBLISH_DEFAULT_ATTRIBUTES = arrayOf(PUBLISH.toString()) +fun addPublishAttributeByDefault(vararg attributes: String): Array { + if (attributes.isNotEmpty()) + return attributes + return PUBLISH_DEFAULT_ATTRIBUTES } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index 853e44559..65b7edf50 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -21,7 +21,7 @@ import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.MessageRouterContext import com.exactpro.th2.common.schema.message.SubscriberMonitor -import com.exactpro.th2.common.schema.message.addSubscribeAttributeByDefault +import com.exactpro.th2.common.schema.message.addPublishAttributeByDefault import com.exactpro.th2.common.schema.message.appendAttributes abstract class AbstractGroupBatchAdapterRouter : MessageRouter { @@ -42,7 +42,7 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { } override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { - return groupBatchRouter.subscribeWithManualAck({ deliveryMetadata: DeliveryMetadata, message: MessageGroupBatch, _ -> + return groupBatchRouter.subscribe({ deliveryMetadata: DeliveryMetadata, message: MessageGroupBatch -> callback.handle(deliveryMetadata, buildFromGroupBatch(message)) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() @@ -50,16 +50,15 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { } override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { - val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) return groupBatchRouter.subscribeAll({ deliveryMetadata: DeliveryMetadata, message: MessageGroupBatch -> callback.handle(deliveryMetadata, buildFromGroupBatch(message)) }, - *appendAttributes(*attributesWithDefaultValue) { getRequiredSubscribeAttributes() }.toTypedArray() + *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) } override fun send(messageBatch: T, vararg attributes: String) { - val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) + val attributesWithDefaultValue = addPublishAttributeByDefault(*attributes) groupBatchRouter.send( buildGroupBatch(messageBatch), *appendAttributes(*attributesWithDefaultValue) { getRequiredSendAttributes() }.toTypedArray() @@ -67,7 +66,7 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { } override fun sendAll(messageBatch: T, vararg attributes: String) { - val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) + val attributesWithDefaultValue = addPublishAttributeByDefault(*attributes) groupBatchRouter.sendAll( buildGroupBatch(messageBatch), *appendAttributes(*attributesWithDefaultValue) { getRequiredSendAttributes() }.toTypedArray() From ebd05269b83012b7aeb24d796734bae15f372b7d Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 3 Nov 2022 16:17:13 +0400 Subject: [PATCH 075/154] [TH2-4299] removed default values --- .../schema/message/MessageRouterUtils.kt | 15 --------------- .../AbstractGroupBatchAdapterRouter.kt | 7 ++----- .../impl/rabbitmq/AbstractRabbitRouter.kt | 19 +++---------------- 3 files changed, 5 insertions(+), 36 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index c7ad4f69a..8480e6e8e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -30,7 +30,6 @@ import com.exactpro.th2.common.message.logId import com.exactpro.th2.common.message.toJson import com.exactpro.th2.common.schema.message.QueueAttribute.EVENT import com.exactpro.th2.common.schema.message.QueueAttribute.PUBLISH -import com.exactpro.th2.common.schema.message.QueueAttribute.SUBSCRIBE import com.google.protobuf.MessageOrBuilder import org.apache.commons.lang3.exception.ExceptionUtils @@ -102,18 +101,4 @@ fun MessageGroupBatch.toShortDebugString(): String = buildString { .apply { append(this) } append(')') -} - -private val SUBSCRIBE_DEFAULT_ATTRIBUTES = arrayOf(SUBSCRIBE.toString()) -fun addSubscribeAttributeByDefault(vararg attributes: String): Array { - if (attributes.isNotEmpty()) - return attributes - return SUBSCRIBE_DEFAULT_ATTRIBUTES -} - -private val PUBLISH_DEFAULT_ATTRIBUTES = arrayOf(PUBLISH.toString()) -fun addPublishAttributeByDefault(vararg attributes: String): Array { - if (attributes.isNotEmpty()) - return attributes - return PUBLISH_DEFAULT_ATTRIBUTES } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index 65b7edf50..3ce3cef16 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -21,7 +21,6 @@ import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.MessageRouterContext import com.exactpro.th2.common.schema.message.SubscriberMonitor -import com.exactpro.th2.common.schema.message.addPublishAttributeByDefault import com.exactpro.th2.common.schema.message.appendAttributes abstract class AbstractGroupBatchAdapterRouter : MessageRouter { @@ -58,18 +57,16 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { } override fun send(messageBatch: T, vararg attributes: String) { - val attributesWithDefaultValue = addPublishAttributeByDefault(*attributes) groupBatchRouter.send( buildGroupBatch(messageBatch), - *appendAttributes(*attributesWithDefaultValue) { getRequiredSendAttributes() }.toTypedArray() + *appendAttributes(*attributes) { getRequiredSendAttributes() }.toTypedArray() ) } override fun sendAll(messageBatch: T, vararg attributes: String) { - val attributesWithDefaultValue = addPublishAttributeByDefault(*attributes) groupBatchRouter.sendAll( buildGroupBatch(messageBatch), - *appendAttributes(*attributesWithDefaultValue) { getRequiredSendAttributes() }.toTypedArray() + *appendAttributes(*attributes) { getRequiredSendAttributes() }.toTypedArray() ) } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index d22c0e10e..b7d931589 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -27,7 +27,6 @@ import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.QueueAttribute.PUBLISH import com.exactpro.th2.common.schema.message.QueueAttribute.SUBSCRIBE import com.exactpro.th2.common.schema.message.SubscriberMonitor -import com.exactpro.th2.common.schema.message.addSubscribeAttributeByDefault import com.exactpro.th2.common.schema.message.appendAttributes import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration @@ -98,7 +97,7 @@ abstract class AbstractRabbitRouter : MessageRouter { override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor { val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } - return subscribe(attributes = attributes, ConfirmationListener.wrap(callback)) { + return subscribe(pintAttributes = pintAttributes, ConfirmationListener.wrap(callback)) { check(size == 1) { "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe operation by attributes $pintAttributes and filters, expected 1, actual $size" } @@ -106,8 +105,7 @@ abstract class AbstractRabbitRouter : MessageRouter { } override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor { - val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) - val pintAttributes: Set = appendAttributes(*attributesWithDefaultValue) { getRequiredSubscribeAttributes() } + val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } val listener = ConfirmationListener.wrap(callback) return subscribe(pintAttributes = pintAttributes, listener) { check(isNotEmpty()) { @@ -126,8 +124,7 @@ abstract class AbstractRabbitRouter : MessageRouter { } override fun subscribeAllWithManualAck(callback: ManualConfirmationListener, vararg attributes: String): SubscriberMonitor { - val attributesWithDefaultValue = addSubscribeAttributeByDefault(*attributes) - val pintAttributes: Set = appendAttributes(*attributesWithDefaultValue) { getRequiredSubscribeAttributes() } + val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } return subscribe(pintAttributes, callback) { check(isNotEmpty()) { "Found incorrect number of pins ${map(PinInfo::pinName)} to subscribe all operation by attributes $pintAttributes and filters, expected 1 or more, actual $size" @@ -204,16 +201,6 @@ abstract class AbstractRabbitRouter : MessageRouter { checkOrThrow(exceptions.values) { "Can't send to pin(s): ${exceptions.keys}" } } - private fun subscribe( - vararg attributes: String, - callback: ConfirmationListener, - check: List.() -> Unit - ): SubscriberMonitor { - val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSubscribeAttributes() } - - return subscribe(pintAttributes, callback, check) - } - private fun subscribe( pintAttributes: Set, messageListener: ConfirmationListener, From 1efce7efb7f928bbf9253ae1159b2bdd954b1e59 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 7 Nov 2022 12:14:14 +0400 Subject: [PATCH 076/154] [TH2-4299] Added MetricsUtilsBenchmark.kt --- Dockerfile | 2 +- build.gradle | 4 + gradle/wrapper/gradle-wrapper.properties | 2 +- src/jmh/kotlin/MetricsUtilsBenchmark.kt | 225 ++++++++++++++++++ .../th2/common/metrics/MetricsUtils.kt | 68 +++++- 5 files changed, 296 insertions(+), 5 deletions(-) create mode 100644 src/jmh/kotlin/MetricsUtilsBenchmark.kt diff --git a/Dockerfile b/Dockerfile index a49de588e..c7e2c2e67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM gradle:6.6-jdk11 AS build +FROM gradle:7.5.1-jdk11 AS build ARG release_version ARG bintray_user ARG bintray_key diff --git a/build.gradle b/build.gradle index 5eab3f182..9cc0be145 100644 --- a/build.gradle +++ b/build.gradle @@ -22,6 +22,7 @@ plugins { id 'com.google.protobuf' version '0.8.8' apply false id 'org.jetbrains.kotlin.jvm' version "${kotlin_version}" id "org.owasp.dependencycheck" version "7.2.0" + id "me.champeau.jmh" version "0.6.8" } group = 'com.exactpro.th2' @@ -174,6 +175,9 @@ dependencies { } api 'com.exactpro.th2:grpc-common:3.11.1' + jmh 'org.openjdk.jmh:jmh-core:0.9' + jmh 'org.openjdk.jmh:jmh-generator-annprocess:0.9' + implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.2.2' implementation ("com.exactpro.th2:cradle-cassandra:${cradleVersion}") { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 73500df24..7d1733249 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ #Thu Jul 02 11:31:27 GMT+04:00 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/src/jmh/kotlin/MetricsUtilsBenchmark.kt b/src/jmh/kotlin/MetricsUtilsBenchmark.kt new file mode 100644 index 000000000..41c374ef5 --- /dev/null +++ b/src/jmh/kotlin/MetricsUtilsBenchmark.kt @@ -0,0 +1,225 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2 + +import com.exactpro.th2.common.grpc.AnyMessage +import com.exactpro.th2.common.grpc.Direction +import com.exactpro.th2.common.grpc.MessageGroup +import com.exactpro.th2.common.grpc.MessageGroupBatch +import com.exactpro.th2.common.metrics.DIRECTION_LABEL +import com.exactpro.th2.common.metrics.MESSAGE_TYPE_LABEL +import com.exactpro.th2.common.metrics.SESSION_ALIAS_LABEL +import com.exactpro.th2.common.metrics.TH2_PIN_LABEL +import com.exactpro.th2.common.metrics.incrementDroppedMetrics +import com.exactpro.th2.common.metrics.incrementTotalMetrics +import com.exactpro.th2.common.metrics.incrementTotalMetricsOld +import io.prometheus.client.Counter +import io.prometheus.client.Gauge +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Mode.AverageTime +import org.openjdk.jmh.annotations.Mode.Throughput +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Scope.Thread +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State + +@State(Scope.Benchmark) +open class MetricsUtilsBenchmark { + + + @State(Thread) + open class BatchState { + lateinit var batch: MessageGroupBatch + lateinit var messages: List + + lateinit var messageCounter: Counter + lateinit var groupCounter: Counter + lateinit var groupSequenceGauge: Gauge + + @Setup + open fun init() { + messages = batch.groupsList.asSequence() + .map(MessageGroup::getMessagesList) + .flatMap(List::asSequence) + .toList() + + messageCounter = Counter.build("message_counter", "Message counter") + .labelNames(TH2_PIN_LABEL, SESSION_ALIAS_LABEL, DIRECTION_LABEL, MESSAGE_TYPE_LABEL) + .register() + + groupCounter = Counter.build("group_counter", "Group counter") + .labelNames(TH2_PIN_LABEL, SESSION_ALIAS_LABEL, DIRECTION_LABEL) + .register() + + groupSequenceGauge = Gauge.build("group_sequence", "Group sequence") + .labelNames(TH2_PIN_LABEL, SESSION_ALIAS_LABEL, DIRECTION_LABEL) + .register() + + println("groups ${batch.groupsCount}, messages ${batch.groupsList.asSequence().map(MessageGroup::getMessagesList).map(List::size).sum()}") + } + + } + + open class Simple: BatchState() { + + override fun init() { + batch = MessageGroupBatch.newBuilder().apply { + repeat(GROUP_IN_BATCH) { + addGroupsBuilder().apply { + repeat(MESSAGES_IN_GROUP) { + addMessagesBuilder().apply { + rawMessageBuilder.apply { + metadataBuilder.apply { + idBuilder.apply { + direction = Direction.FIRST + sequence = SEQUENCE_GENERATOR.next() + connectionIdBuilder.apply { + sessionAlias = ALIAS + } + } + } + } + } + } + } + } + }.build() + super.init() + } + } + + open class Multiple: BatchState() { + override fun init() { + batch = MessageGroupBatch.newBuilder().apply { + repeat(GROUP_IN_BATCH) { + addGroupsBuilder().apply { + repeat(MESSAGES_IN_GROUP) { + addMessagesBuilder().apply { + rawMessageBuilder.apply { + metadataBuilder.apply { + idBuilder.apply { + direction = DIRECTION_GENERATOR.next() + sequence = SEQUENCE_GENERATOR.next() + connectionIdBuilder.apply { + sessionAlias = ALIAS_GENERATOR.next() + } + } + } + } + } + } + } + } + }.build() + super.init() + } + } + + @Benchmark + @BenchmarkMode(Throughput, AverageTime) + fun benchmarkIncrementTotalMetricsOldVsSimpleBatch(state: Simple) { + incrementTotalMetricsOld( + state.batch, + TH2_PIN, + state.messageCounter, + state.groupCounter, + state.groupSequenceGauge + ) + } + + @Benchmark + @BenchmarkMode(Throughput, AverageTime) + fun benchmarkIncrementTotalMetricsOldVsMultipleBatch(state: Multiple) { + incrementTotalMetricsOld( + state.batch, + TH2_PIN, + state.messageCounter, + state.groupCounter, + state.groupSequenceGauge + ) + } + + @Benchmark + @BenchmarkMode(Throughput, AverageTime) + fun benchmarkIncrementTotalMetricsVsSimpleBatch(state: Simple) { + incrementTotalMetrics( + state.batch, + TH2_PIN, + state.messageCounter, + state.groupCounter, + state.groupSequenceGauge + ) + } + + @Benchmark + @BenchmarkMode(Throughput, AverageTime) + fun benchmarkIncrementTotalMetricsVsMultipleBatch(state: Multiple) { + incrementTotalMetrics( + state.batch, + TH2_PIN, + state.messageCounter, + state.groupCounter, + state.groupSequenceGauge + ) + } + + @Benchmark + @BenchmarkMode(Throughput, AverageTime) + fun benchmarkIncrementDroppedMetricsVsSimpleBatch(state: Simple) { + incrementDroppedMetrics( + state.messages, + TH2_PIN, + state.messageCounter, + state.groupCounter, + ) + } + + @Benchmark + @BenchmarkMode(Throughput, AverageTime) + fun benchmarkIncrementDroppedMetricsVsMultipleBatch(state: Multiple) { + incrementDroppedMetrics( + state.messages, + TH2_PIN, + state.messageCounter, + state.groupCounter, + ) + } + + companion object { + private const val TH2_PIN = "pin" + private const val ALIAS = "alias" + + private const val ALIASES = 100 + private const val GROUP_IN_BATCH = 3_000 + private const val MESSAGES_IN_GROUP = 1 + + private val SEQUENCE_GENERATOR = generateSequence(1L) { it + 1 } + .iterator() + + private val ALIAS_GENERATOR = sequence { + var counter = 0 + while (true) { + yield(counter++ % ALIASES + 1) + } + }.map { ALIAS + it } + .iterator() + + private val DIRECTION_GENERATOR = generateSequence(1) { it + 1 } + .map { if (it % 2 == 0) Direction.FIRST else Direction.SECOND } + .iterator() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt b/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt index 1c77579be..670ca8929 100644 --- a/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt @@ -17,21 +17,25 @@ package com.exactpro.th2.common.metrics import com.exactpro.th2.common.grpc.AnyMessage +import com.exactpro.th2.common.grpc.AnyMessage.KindCase.MESSAGE +import com.exactpro.th2.common.grpc.AnyMessage.KindCase.RAW_MESSAGE +import com.exactpro.th2.common.grpc.Direction.FIRST +import com.exactpro.th2.common.grpc.Direction.SECOND import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.message.getSessionAliasAndDirection import com.exactpro.th2.common.message.sequence import io.prometheus.client.Counter import io.prometheus.client.Gauge +import kotlin.math.max -fun incrementTotalMetrics( + +fun incrementTotalMetricsOld( batch: MessageGroupBatch, th2Pin: String, messageCounter: Counter, groupCounter: Counter, gauge: Gauge ) { - if (true) return //FIXME: added for test - val groupsBySessionAliasAndDirection = mutableMapOf() batch.groupsList.forEach { group -> if (group.messagesList.isNotEmpty()) { @@ -61,6 +65,64 @@ fun incrementTotalMetrics( } } +data class SessionStats( + var sequence: Long = 0L, + var groups: Int = 0, + var messages: Int = 0, + var rawMessages: Int = 0, +) + +fun incrementTotalMetrics( + batch: MessageGroupBatch, + th2Pin: String, + messageCounter: Counter, + groupCounter: Counter, + gauge: Gauge, +) { + val incomingStatsBySession = mutableMapOf() + val outgoingStatsBySession = mutableMapOf() + + for (group in batch.groupsList) { + val messages = group.messagesList + + if (messages.isEmpty()) continue + + val firstMessage = messages[0] + val id = if (firstMessage.hasMessage()) firstMessage.message.metadata.id else firstMessage.rawMessage.metadata.id + val sessionAlias = id.connectionId.sessionAlias + + val stats = when (id.direction) { + FIRST -> incomingStatsBySession.getOrPut(sessionAlias, ::SessionStats) + else -> outgoingStatsBySession.getOrPut(sessionAlias, ::SessionStats) + } + + stats.groups++ + + messages.forEach { + when { + it.hasMessage() -> stats.messages++ + else -> stats.rawMessages++ + } + } + + stats.sequence = max(stats.sequence, id.sequence) + } + + incomingStatsBySession.forEach { (alias, stats) -> + gauge.labels(th2Pin, alias, FIRST.name).set(stats.sequence.toDouble()) + groupCounter.labels(th2Pin, alias, FIRST.name).inc(stats.groups.toDouble()) + messageCounter.labels(th2Pin, alias, FIRST.name, MESSAGE.name).inc(stats.messages.toDouble()) + messageCounter.labels(th2Pin, alias, FIRST.name, RAW_MESSAGE.name).inc(stats.rawMessages.toDouble()) + } + + outgoingStatsBySession.forEach { (alias, stats) -> + gauge.labels(th2Pin, alias, SECOND.name).set(stats.sequence.toDouble()) + groupCounter.labels(th2Pin, alias, SECOND.name).inc(stats.groups.toDouble()) + messageCounter.labels(th2Pin, alias, SECOND.name, MESSAGE.name).inc(stats.messages.toDouble()) + messageCounter.labels(th2Pin, alias, SECOND.name, RAW_MESSAGE.name).inc(stats.rawMessages.toDouble()) + } +} + fun incrementDroppedMetrics( messages: List, th2Pin: String, From a61cb0c89654e13a8680a9a97f414b26728913c7 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 15 Nov 2022 19:55:21 +0400 Subject: [PATCH 077/154] [TH2-4443] Implemented sendExclusive and subscribeExclusive methods --- .../grpc/router/AbstractGrpcRouter.java | 2 +- .../common/schema/message/MessageRouter.java | 22 ++- .../schema/message/MessageSubscriber.java | 11 +- .../schema/message/SubscriberMonitor.java | 6 +- .../rabbitmq/AbstractRabbitSubscriber.java | 168 ++++++++++-------- .../connection/ConnectionManager.java | 98 ++++++---- .../message/SubscriberExclusiveMonitor.kt | 3 +- .../AbstractGroupBatchAdapterRouter.kt | 24 ++- .../impl/rabbitmq/AbstractRabbitRouter.kt | 30 +++- .../TestRabbitMessageGroupBatchRouter.kt | 4 +- 10 files changed, 244 insertions(+), 124 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java index cea110207..792e869d2 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java @@ -228,7 +228,7 @@ protected Function createGetMeasuringMetric(Counte return (MethodDetails) -> null; } - protected void throwIsInit() { + protected void failIfInitialized() { if (this.configuration != null) { throw new IllegalStateException("Grpc router already init"); } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index a7de45b68..2f9882374 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -33,7 +33,6 @@ public interface MessageRouter extends AutoCloseable { /** * Initialization message router - * @param connectionManager * @param configuration message router configuration */ @Deprecated(since = "3.2.2", forRemoval = true) @@ -54,6 +53,14 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< */ void init(@NotNull MessageRouterContext context); + /** + * Create new exclusive queue and subscribe to it. Only own connection can use the particular queue. + * Please note Exclusive queues are deleted when their declaring connection is closed or gone (e.g. due to underlying TCP connection loss). + * They therefore are only suitable for client-specific transient state. + * @return {@link SubscriberExclusiveMonitor} object to manage subscription. + */ + SubscriberExclusiveMonitor subscribeExclusive(MessageListener callback); + /** * Listen ONE RabbitMQ queue by intersection schemas queues attributes * @param callback listener @@ -113,10 +120,15 @@ default SubscriberMonitor subscribeAllWithManualAck(ConfirmationMessageListener< throw new UnsupportedOperationException("The subscription with manual confirmation is not supported"); } + /** + * Send the message to the queue + * @throws IOException if router can not send message + */ + void sendExclusive(String queue, T message) throws IOException; + /** * Send message to SOME RabbitMQ queues which match the filter for this message - * @param message - * @throws IOException if can not send message + * @throws IOException if router can not send message */ default void send(T message) throws IOException { send(message, QueueAttribute.PUBLISH.toString()); @@ -124,7 +136,6 @@ default void send(T message) throws IOException { /** * Send message to ONE RabbitMQ queue by intersection schemas queues attributes - * @param message * @param queueAttr schemas queues attributes * @throws IOException if can not send message * @throws IllegalStateException when more than 1 queue is found @@ -133,7 +144,6 @@ default void send(T message) throws IOException { /** * Send message to SOME RabbitMQ queue by intersection schemas queues attributes - * @param message * @param queueAttr schemas queues attributes * @throws IOException if can not send message */ diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java index 1b57f96bd..a24737117 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java @@ -20,12 +20,12 @@ import org.jetbrains.annotations.NotNull; -import javax.annotation.concurrent.NotThreadSafe; +import javax.annotation.concurrent.ThreadSafe; /** * Listen messages and transmit it to {@link MessageListener} */ -@NotThreadSafe +@ThreadSafe public interface MessageSubscriber extends AutoCloseable { // Please use constructor for initialization @Deprecated(since = "3.3.0", forRemoval = true) @@ -35,7 +35,14 @@ public interface MessageSubscriber extends AutoCloseable { @Deprecated void init(@NotNull ConnectionManager connectionManager, @NotNull SubscribeTarget subscribeTarget, @NotNull FilterFunction filterFunc); + /** + * @deprecated subscriber automatically subscribe after adding first listener and + * unsubscribe after remove the last one + */ + @Deprecated void start() throws Exception; void addListener(ConfirmationMessageListener messageListener); + + void removeListener(ConfirmationMessageListener messageListener); } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/SubscriberMonitor.java b/src/main/java/com/exactpro/th2/common/schema/message/SubscriberMonitor.java index 1609018d5..86c882057 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/SubscriberMonitor.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/SubscriberMonitor.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,10 @@ package com.exactpro.th2.common.schema.message; +import java.io.IOException; + public interface SubscriberMonitor { - void unsubscribe() throws Exception; + void unsubscribe() throws IOException; } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index ad44cc30f..b885545cf 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -38,10 +38,11 @@ import java.io.IOException; import java.util.List; +import java.util.ListIterator; import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReentrantLock; import static com.exactpro.th2.common.metrics.CommonMetrics.DEFAULT_BUCKETS; import static com.exactpro.th2.common.metrics.CommonMetrics.QUEUE_LABEL; @@ -68,14 +69,16 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber> listeners = new CopyOnWriteArrayList<>(); + private final CopyOnWriteArrayList> listeners = new CopyOnWriteArrayList<>(); private final String queue; - private final AtomicReference connectionManager = new AtomicReference<>(); + private final ConnectionManager connectionManager; private final AtomicReference consumerMonitor = new AtomicReference<>(); private final AtomicReference filterFunc = new AtomicReference<>(); private final String th2Type; - private final AtomicBoolean hasManualSubscriber = new AtomicBoolean(); private final HealthMetrics healthMetrics = new HealthMetrics(this); @@ -86,7 +89,7 @@ public AbstractRabbitSubscriber( @NotNull String th2Pin, @NotNull String th2Type ) { - this.connectionManager.set(requireNonNull(connectionManager, "Connection can not be null")); + this.connectionManager = requireNonNull(connectionManager, "Connection can not be null"); this.queue = requireNonNull(queue, "Queue can not be null"); this.filterFunc.set(requireNonNull(filterFunc, "Filter function can not be null")); this.th2Pin = requireNonNull(th2Pin, "TH2 pin can not be null"); @@ -106,85 +109,63 @@ public void init(@NotNull ConnectionManager connectionManager, @NotNull Subscrib } @Override + @Deprecated public void start() throws Exception { - ConnectionManager connectionManager = this.connectionManager.get(); - if (connectionManager == null) { - throw new IllegalStateException("Subscriber is not initialized"); - } + // Do nothing + } + @Override + public void addListener(ConfirmationMessageListener messageListener) { try { - consumerMonitor.updateAndGet(monitor -> { - if (monitor == null) { - try { - monitor = connectionManager.basicConsume( - queue, - (consumeTag, delivery, confirmation) -> { - Timer processTimer = MESSAGE_PROCESS_DURATION_SECONDS - .labels(th2Pin, th2Type, queue) - .startTimer(); - MESSAGE_SIZE_SUBSCRIBE_BYTES - .labels(th2Pin, th2Type, queue) - .inc(delivery.getBody().length); - try { - T value; - try { - value = valueFromBytes(delivery.getBody()); - } catch (Exception e) { - LOGGER.error("Couldn't parse delivery. Confirm message received", e); - confirmation.confirm(); - throw new IOException( - String.format( - "Can not extract value from bytes for envelope '%s', queue '%s', pin '%s'", - delivery.getEnvelope(), queue, th2Pin - ), - e - ); - } - handle(consumeTag, delivery, value, confirmation); - } finally { - processTimer.observeDuration(); - } - }, - this::canceled - ); - LOGGER.info("Start listening queue name='{}'", queue); - } catch (IOException e) { - throw new IllegalStateException("Can not start subscribe to queue = " + queue, e); - } + publicLock.lock(); + boolean isManual = ConfirmationMessageListener.isManual(messageListener); + if (isManual) { + if (hasManualSubscriber) { + throw new IllegalStateException("cannot subscribe listener " + messageListener + + " because only one listener with manual confirmation is allowed per queue"); } - - return monitor; - }); - } catch (Exception e) { - throw new IllegalStateException("Can not start listening", e); + } + if(listeners.addIfAbsent(messageListener)) { + hasManualSubscriber |= isManual; + subscribe(); + } + } finally { + publicLock.unlock(); } } @Override - public void addListener(ConfirmationMessageListener messageListener) { - if (ConfirmationMessageListener.isManual(messageListener)) { - if (!hasManualSubscriber.compareAndSet(false, true)) { - throw new IllegalStateException("cannot subscribe listener " + messageListener - + " because only one listener with manual confirmation is allowed per queue"); + public void removeListener(ConfirmationMessageListener messageListener) { + try { + publicLock.lock(); + boolean isManual = ConfirmationMessageListener.isManual(messageListener); + if (listeners.remove(messageListener)) { + hasManualSubscriber = !(hasManualSubscriber && isManual); + messageListener.onClose(); } + } finally { + publicLock.unlock(); } - listeners.add(messageListener); } @Override - public void close() throws Exception { - ConnectionManager connectionManager = this.connectionManager.get(); - if (connectionManager == null) { - throw new IllegalStateException("Subscriber is not initialized"); - } + public void close() throws IOException { + try { + publicLock.lock(); + SubscriberMonitor monitor = consumerMonitor.getAndSet(null); + if (monitor != null) { + monitor.unsubscribe(); + } - SubscriberMonitor monitor = consumerMonitor.getAndSet(null); - if (monitor != null) { - monitor.unsubscribe(); + ListIterator> listIterator = listeners.listIterator(); + while (listIterator.hasNext()) { + ConfirmationMessageListener listener = listIterator.next(); + listIterator.remove(); + listener.onClose(); + } + } finally { + publicLock.unlock(); } - - listeners.forEach(ConfirmationMessageListener::onClose); - listeners.clear(); } protected boolean callFilterFunction(Message message, List filters) { @@ -248,6 +229,53 @@ protected void handle(String consumeTag, Delivery delivery, T value, Confirmatio } } + private void subscribe() { + try { + consumerMonitor.updateAndGet(monitor -> { + if (monitor == null) { + try { + monitor = connectionManager.basicConsume( + queue, + (consumeTag, delivery, confirmation) -> { + try (Timer ignored = MESSAGE_PROCESS_DURATION_SECONDS + .labels(th2Pin, th2Type, queue) + .startTimer()) { + MESSAGE_SIZE_SUBSCRIBE_BYTES + .labels(th2Pin, th2Type, queue) + .inc(delivery.getBody().length); + + T value; + try { + value = valueFromBytes(delivery.getBody()); + } catch (Exception e) { + LOGGER.error("Couldn't parse delivery. Confirm message received", e); + confirmation.confirm(); + throw new IOException( + String.format( + "Can not extract value from bytes for envelope '%s', queue '%s', pin '%s'", + delivery.getEnvelope(), queue, th2Pin + ), + e + ); + } + handle(consumeTag, delivery, value, confirmation); + } + }, + this::canceled + ); + LOGGER.info("Start listening queue name='{}', th2 pin='{}'", queue, th2Pin); + } catch (IOException e) { + throw new IllegalStateException("Can not start subscribe to queue = " + queue, e); + } + } + + return monitor; + }); + } catch (Exception e) { + throw new IllegalStateException("Can not start listening", e); + } + } + private void resubscribe() { LOGGER.info("Try to resubscribe subscriber for queue name='{}'", queue); @@ -261,7 +289,7 @@ private void resubscribe() { } try { - start(); + subscribe(); } catch (Exception e) { LOGGER.error("Can not resubscribe subscriber for queue name='{}'", queue); healthMetrics.disable(); diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java index 656f6d34e..8cc0e438d 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -34,6 +34,7 @@ import javax.annotation.concurrent.GuardedBy; +import com.exactpro.th2.common.schema.message.SubscriberExclusiveMonitor; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -47,7 +48,6 @@ import com.exactpro.th2.common.metrics.HealthMetrics; import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback; import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; -import com.exactpro.th2.common.schema.message.SubscriberMonitor; import com.exactpro.th2.common.schema.message.impl.OnlyOnceConfirmation; import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration; import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration; @@ -84,19 +84,6 @@ public class ConnectionManager implements AutoCloseable { private final HealthMetrics metrics = new HealthMetrics(this); - private final RecoveryListener recoveryListener = new RecoveryListener() { - @Override - public void handleRecovery(Recoverable recoverable) { - LOGGER.debug("Count tries to recovery connection reset to 0"); - connectionRecoveryAttempts.set(0); - metrics.getReadinessMonitor().enable(); - LOGGER.debug("Set RabbitMQ readiness to true"); - } - - @Override - public void handleRecoveryStarted(Recoverable recoverable) {} - }; - public ConnectionManagerConfiguration getConfiguration() { return configuration; } @@ -237,18 +224,31 @@ private void turnOffReadiness(Throwable exception){ this.connection.addBlockedListener(new BlockedListener() { @Override - public void handleBlocked(String reason) throws IOException { + public void handleBlocked(String reason) { LOGGER.warn("RabbitMQ blocked connection: {}", reason); } @Override - public void handleUnblocked() throws IOException { + public void handleUnblocked() { LOGGER.warn("RabbitMQ unblocked connection"); } }); if (this.connection instanceof Recoverable) { Recoverable recoverableConnection = (Recoverable) this.connection; + RecoveryListener recoveryListener = new RecoveryListener() { + @Override + public void handleRecovery(Recoverable recoverable) { + LOGGER.debug("Count tries to recovery connection reset to 0"); + connectionRecoveryAttempts.set(0); + metrics.getReadinessMonitor().enable(); + LOGGER.debug("Set RabbitMQ readiness to true"); + } + + @Override + public void handleRecoveryStarted(Recoverable recoverable) { + } + }; recoverableConnection.addRecoveryListener(recoveryListener); LOGGER.debug("Recovery listener was added to connection."); } else { @@ -285,11 +285,21 @@ public void close() { } public void basicPublish(String exchange, String routingKey, BasicProperties props, byte[] body) throws IOException { - ChannelHolder holder = getChannelFor(PinId.forRoutingKey(routingKey)); + ChannelHolder holder = getChannelFor(PinId.forRoutingKey(exchange, routingKey)); holder.withLock(channel -> channel.basicPublish(exchange, routingKey, props, body)); } - public SubscriberMonitor basicConsume(String queue, ManualAckDeliveryCallback deliverCallback, CancelCallback cancelCallback) throws IOException { + public String queueDeclare() throws IOException { + ChannelHolder holder = new ChannelHolder(this::createChannel, this::waitForConnectionRecovery, configuration.getPrefetchCount()); + return holder.mapWithLock( channel -> { + String queue = holder.channel.queueDeclare().getQueue(); + LOGGER.info("Declared exclusive '{}' queue", queue); + putChannelFor(PinId.forQueue(queue), holder); + return queue; + }); + } + + public SubscriberExclusiveMonitor basicConsume(String queue, ManualAckDeliveryCallback deliverCallback, CancelCallback cancelCallback) throws IOException { ChannelHolder holder = getChannelFor(PinId.forQueue(queue)); String tag = holder.mapWithLock(channel -> channel.basicConsume(queue, false, subscriberName + "_" + nextSubscriberId.getAndIncrement(), (tagTmp, delivery) -> { @@ -325,7 +335,7 @@ public SubscriberMonitor basicConsume(String queue, ManualAckDeliveryCallback de } }, cancelCallback)); - return new RabbitMqSubscriberMonitor(holder, tag, this::basicCancel); + return new RabbitMqSubscriberMonitor(holder, queue, tag, this::basicCancel); } boolean isReady() { @@ -360,6 +370,13 @@ private ChannelHolder getChannelFor(PinId pinId) { }); } + private void putChannelFor(PinId pinId, ChannelHolder holder) { + ChannelHolder previous = channelsByPin.putIfAbsent(pinId, holder); + if (previous != null) { + throw new IllegalStateException("Channel holder for the '" + pinId + "' pinId has been already registered"); + } + } + private Channel createChannel() { waitForConnectionRecovery(connection); @@ -413,27 +430,35 @@ private boolean isConnectionRecovery(ShutdownNotifier notifier) { /** * @param channel pass channel witch used for basicConsume, because delivery tags are scoped per channel, * deliveries must be acknowledged on the same channel they were received on. - * @throws IOException */ private static void basicAck(Channel channel, long deliveryTag) throws IOException { channel.basicAck(deliveryTag, false); } - private static class RabbitMqSubscriberMonitor implements SubscriberMonitor { + private static class RabbitMqSubscriberMonitor implements SubscriberExclusiveMonitor { private final ChannelHolder holder; + private final String queue; private final String tag; private final CancelAction action; - public RabbitMqSubscriberMonitor(ChannelHolder holder, String tag, + public RabbitMqSubscriberMonitor(ChannelHolder holder, + String queue, + String tag, CancelAction action) { this.holder = holder; + this.queue = queue; this.tag = tag; this.action = action; } @Override - public void unsubscribe() throws Exception { + public @NotNull String getQueue() { + return queue; + } + + @Override + public void unsubscribe() throws IOException { holder.withLock(false, channel -> action.execute(channel, tag)); } } @@ -443,21 +468,23 @@ private interface CancelAction { } private static class PinId { + private final String exchange; private final String routingKey; private final String queue; - public static PinId forRoutingKey(String routingKey) { - return new PinId(routingKey, null); + public static PinId forRoutingKey(String exchange, String routingKey) { + return new PinId(exchange, routingKey, null); } public static PinId forQueue(String queue) { - return new PinId(null, queue); + return new PinId(null, null, queue); } - private PinId(String routingKey, String queue) { - if (routingKey == null && queue == null) { - throw new NullPointerException("Either routingKey or queue must be set"); + private PinId(String exchange, String routingKey, String queue) { + if ((exchange == null || routingKey == null) && queue == null) { + throw new NullPointerException("Either exchange and routingKey or queue must be set"); } + this.exchange = exchange; this.routingKey = routingKey; this.queue = queue; } @@ -470,17 +497,24 @@ public boolean equals(Object o) { PinId pinId = (PinId) o; - return new EqualsBuilder().append(routingKey, pinId.routingKey).append(queue, pinId.queue).isEquals(); + return new EqualsBuilder() + .append(exchange, pinId.exchange) + .append(routingKey, pinId.routingKey) + .append(queue, pinId.queue).isEquals(); } @Override public int hashCode() { - return new HashCodeBuilder(17, 37).append(routingKey).append(queue).toHashCode(); + return new HashCodeBuilder(17, 37) + .append(exchange) + .append(routingKey) + .append(queue).toHashCode(); } @Override public String toString() { return new ToStringBuilder(this, ToStringStyle.JSON_STYLE) + .append("exchange", exchange) .append("routingKey", routingKey) .append("queue", queue) .toString(); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/SubscriberExclusiveMonitor.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/SubscriberExclusiveMonitor.kt index 25f798706..bd05d211e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/SubscriberExclusiveMonitor.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/SubscriberExclusiveMonitor.kt @@ -16,5 +16,6 @@ package com.exactpro.th2.common.schema.message -interface SubscriberExclusiveMonitor { +interface SubscriberExclusiveMonitor : SubscriberMonitor { + val queue: String } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index f2a98ddcd..2d6e88785 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -19,6 +19,7 @@ import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.MessageRouterContext +import com.exactpro.th2.common.schema.message.SubscriberExclusiveMonitor import com.exactpro.th2.common.schema.message.SubscriberMonitor import com.exactpro.th2.common.schema.message.appendAttributes @@ -39,24 +40,37 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { this.groupBatchRouter = groupBatchRouter } - override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { + override fun subscribeExclusive(callback: MessageListener): SubscriberExclusiveMonitor { + return groupBatchRouter.subscribeExclusive { consumerTag: String, message: MessageGroupBatch -> + callback.handler(consumerTag, buildFromGroupBatch(message)) + } + } + + override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor { return groupBatchRouter.subscribe( - MessageListener { consumerTag: String, message: MessageGroupBatch -> + { consumerTag: String, message: MessageGroupBatch -> callback.handler(consumerTag, buildFromGroupBatch(message)) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) } - override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor? { + override fun subscribeAll(callback: MessageListener, vararg attributes: String): SubscriberMonitor { return groupBatchRouter.subscribeAll( - MessageListener { consumerTag: String, message: MessageGroupBatch -> + { consumerTag: String, message: MessageGroupBatch -> callback.handler(consumerTag, buildFromGroupBatch(message)) }, *appendAttributes(*attributes) { getRequiredSubscribeAttributes() }.toTypedArray() ) } + override fun sendExclusive(queue: String, messageBatch: T) { + groupBatchRouter.sendExclusive( + queue, + buildGroupBatch(messageBatch) + ) + } + override fun send(messageBatch: T, vararg attributes: String) { groupBatchRouter.send( buildGroupBatch(messageBatch), diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index 4072d4ca5..34b0c07ea 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -64,6 +64,13 @@ abstract class AbstractRabbitRouter : MessageRouter { this.context = context } + override fun sendExclusive(queue: String, message: T) { + val pinConfig = PinConfiguration(queue, "", "", isReadable = false, isWritable = true) + + senders.getSender(queue, pinConfig) + .send(message) + } + override fun send(message: T, vararg attributes: String) { val pintAttributes: Set = appendAttributes(*attributes) { getRequiredSendAttributes() } send(message, pintAttributes) { @@ -82,6 +89,24 @@ abstract class AbstractRabbitRouter : MessageRouter { } } + override fun subscribeExclusive(callback: MessageListener): SubscriberExclusiveMonitor { + val queue = connectionManager.queueDeclare() + val pinConfig = PinConfiguration("", queue, "", isReadable = true, isWritable = false) + + val messageListener = ConfirmationMessageListener.wrap(callback) + val messageSubscriber = subscribers.getSubscriber(queue, pinConfig).apply { + addListener(messageListener) + } + + return object: SubscriberExclusiveMonitor { + override val queue: String = queue + + override fun unsubscribe() { + messageSubscriber.removeListener(messageListener) + } + } + } + override fun subscribe(callback: MessageListener, vararg attributes: String): SubscriberMonitor { return subscribeWithManualAck(ConfirmationMessageListener.wrap(callback), *attributes) } @@ -195,13 +220,12 @@ abstract class AbstractRabbitRouter : MessageRouter { runCatching { subscribers.getSubscriber(pinName, pinConfig).apply { addListener(messageListener) - start() //TODO: replace to lazy start on add listener(s) } }.onFailure { e -> LOGGER.error(e) { "Listener can't be subscribed via the $pinName pin" } exceptions[pinName] = e - }.onSuccess { - monitors.add(SubscriberMonitor { close() }) + }.onSuccess { messageSubscriber -> + monitors.add(SubscriberMonitor { messageSubscriber.removeListener(messageListener) }) } } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt index 4b48e4510..6fc9fb667 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt @@ -22,7 +22,7 @@ import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.plusAssign import com.exactpro.th2.common.schema.message.MessageRouter -import com.exactpro.th2.common.schema.message.SubscriberMonitor +import com.exactpro.th2.common.schema.message.SubscriberExclusiveMonitor import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration @@ -45,7 +45,7 @@ import org.mockito.kotlin.verify class TestRabbitMessageGroupBatchRouter { private val connectionConfiguration = ConnectionManagerConfiguration() - private val monitor: SubscriberMonitor = mock { } + private val monitor: SubscriberExclusiveMonitor = mock { } private val connectionManager: ConnectionManager = mock { on { configuration }.thenReturn(connectionConfiguration) on { basicConsume(any(), any(), any()) }.thenReturn(monitor) From b8abf10e31eadd7efbd9a1bb7dd8600b8c939ef9 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 16 Nov 2022 19:34:49 +0400 Subject: [PATCH 078/154] [TH2-4443] Added test for queueDeclare in ConnectionManager --- build.gradle | 1 + .../connection/ConnectionManager.java | 13 ++- .../connection/TestConnectionManager.kt | 110 +++++++++++++++++- 3 files changed, 115 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 9cc0be145..36b865720 100644 --- a/build.gradle +++ b/build.gradle @@ -230,6 +230,7 @@ dependencies { testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" testImplementation 'org.mockito.kotlin:mockito-kotlin:4.0.0' + testImplementation 'org.jetbrains.kotlin:kotlin-test-junit5' testImplementation "org.testcontainers:testcontainers:1.17.1" testImplementation "org.testcontainers:rabbitmq:1.17.1" diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java index 8cc0e438d..8bc55b634 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java @@ -15,6 +15,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.connection; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -213,7 +214,8 @@ private void turnOffReadiness(Throwable exception){ factory.setSharedExecutor(sharedExecutor); try { - this.connection = factory.newConnection(); + connection = factory.newConnection(); + LOGGER.info("Created RabbitMQ connection {} [{}]", connection, connection.hashCode()); metrics.getReadinessMonitor().enable(); LOGGER.debug("Set RabbitMQ readiness to true"); } catch (IOException | TimeoutException e) { @@ -292,7 +294,7 @@ public void basicPublish(String exchange, String routingKey, BasicProperties pro public String queueDeclare() throws IOException { ChannelHolder holder = new ChannelHolder(this::createChannel, this::waitForConnectionRecovery, configuration.getPrefetchCount()); return holder.mapWithLock( channel -> { - String queue = holder.channel.queueDeclare().getQueue(); + String queue = holder.channel.queueDeclare("", false, true, false, Collections.emptyMap()).getQueue(); LOGGER.info("Declared exclusive '{}' queue", queue); putChannelFor(PinId.forQueue(queue), holder); return queue; @@ -386,6 +388,7 @@ private Channel createChannel() { channel.basicQos(configuration.getPrefetchCount()); channel.addReturnListener(ret -> LOGGER.warn("Can not router message to exchange '{}', routing key '{}'. Reply code '{}' and text = {}", ret.getExchange(), ret.getRoutingKey(), ret.getReplyCode(), ret.getReplyText())); + LOGGER.info("Created new RabbitMQ channel {} via connection {}", channel.getChannelNumber(), connection.hashCode()); return channel; } catch (IOException e) { throw new IllegalStateException("Can not create channel", e); @@ -568,7 +571,11 @@ public void withLock(boolean waitForRecovery, ChannelConsumer consumer) throws I public T mapWithLock(ChannelMapper mapper) throws IOException { lock.lock(); try { - return mapper.map(getChannel()); + Channel channel = getChannel(); + return mapper.map(channel); + } catch (IOException e) { + LOGGER.error("Operation failure on the {} channel", channel.getChannelNumber(), e); + throw e; } finally { lock.unlock(); } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/TestConnectionManager.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/TestConnectionManager.kt index fe805a0af..66396b795 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/TestConnectionManager.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/TestConnectionManager.kt @@ -20,15 +20,19 @@ import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration import com.rabbitmq.client.BuiltinExchangeType +import com.rabbitmq.client.CancelCallback import mu.KotlinLogging import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.testcontainers.containers.RabbitMQContainer import org.testcontainers.utility.DockerImageName +import java.io.IOException import java.time.Duration import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit +import kotlin.test.assertFailsWith private val LOGGER = KotlinLogging.logger { } @@ -80,20 +84,25 @@ class TestConnectionManager { manager.basicPublish(exchange, routingKey, null, "Hello $index".toByteArray(Charsets.UTF_8)) } - Assertions.assertTrue(countDown.await(1L, TimeUnit.SECONDS)) { "Not all messages were received: ${countDown.count}" } + assertTrue( + countDown.await( + 1L, + TimeUnit.SECONDS + ) + ) { "Not all messages were received: ${countDown.count}" } - Assertions.assertTrue(manager.isAlive) { "Manager should still be alive" } - Assertions.assertTrue(manager.isReady) { "Manager should be ready until the confirmation timeout expires" } + assertTrue(manager.isAlive) { "Manager should still be alive" } + assertTrue(manager.isReady) { "Manager should be ready until the confirmation timeout expires" } Thread.sleep(confirmationTimeout.toMillis() + 100/*just in case*/) // wait for confirmation timeout - Assertions.assertTrue(manager.isAlive) { "Manager should still be alive" } + assertTrue(manager.isAlive) { "Manager should still be alive" } Assertions.assertFalse(manager.isReady) { "Manager should not be ready" } queue.poll().confirm() - Assertions.assertTrue(manager.isAlive) { "Manager should still be alive" } - Assertions.assertTrue(manager.isReady) { "Manager should be ready" } + assertTrue(manager.isAlive) { "Manager should still be alive" } + assertTrue(manager.isReady) { "Manager should be ready" } val receivedData = generateSequence { queue.poll(10L, TimeUnit.MILLISECONDS) } .onEach(ManualAckDeliveryCallback.Confirmation::confirm) @@ -102,4 +111,93 @@ class TestConnectionManager { } } } + + @Test + fun `connection manager exclusive queue test`() { + RabbitMQContainer(DockerImageName.parse("rabbitmq:3.8-management-alpine")) + .use { rabbitMQContainer -> + rabbitMQContainer.start() + LOGGER.info { "Started with port ${rabbitMQContainer.amqpPort}" } + + createConnectionManager(rabbitMQContainer).use { firstManager -> + createConnectionManager(rabbitMQContainer).use { secondManager -> + val queue = firstManager.queueDeclare() + + assertFailsWith("Another connection can subscribe to the $queue queue") { + secondManager.basicConsume(queue, { _, _, _ -> }, {}) + } + + extracted(firstManager, secondManager, queue, 3) + extracted(firstManager, secondManager, queue, 6) + } + } + + } + } + + private fun extracted( + firstManager: ConnectionManager, + secondManager: ConnectionManager, + queue: String, + cycle: Int + ) { + val countDown = CountDownLatch(cycle) + val deliverCallback = ManualAckDeliveryCallback { _, delivery, conformation -> + countDown.countDown() + LOGGER.info { "Received ${delivery.body.toString(Charsets.UTF_8)} from ${delivery.envelope.exchange}:${delivery.envelope.routingKey}, received ${countDown.count}" } + conformation.confirm() + } + val cancelCallback = CancelCallback { LOGGER.warn { "Canceled $it" } } + + val firstMonitor = firstManager.basicConsume(queue, deliverCallback, cancelCallback) + val secondMonitor = firstManager.basicConsume(queue, deliverCallback, cancelCallback) + + repeat(cycle) { index -> + secondManager.basicPublish( + "", + queue, + null, + "Hello $index".toByteArray(Charsets.UTF_8) + ) + } + + assertTrue( + countDown.await( + 1L, + TimeUnit.SECONDS + ) + ) { "Not all messages were received: ${countDown.count}" } + + assertTrue(firstManager.isAlive) { "Manager should still be alive" } + assertTrue(firstManager.isReady) { "Manager should be ready until the confirmation timeout expires" } + + firstMonitor.unsubscribe() + secondMonitor.unsubscribe() + } + + private fun createConnectionManager( + rabbitMQContainer: RabbitMQContainer, + prefetchCount: Int = DEFAULT_PREFETCH_COUNT, + confirmationTimeout: Duration = DEFAULT_CONFIRMATION_TIMEOUT + ) = ConnectionManager( + RabbitMQConfiguration( + host = rabbitMQContainer.host, + vHost = "", + port = rabbitMQContainer.amqpPort, + username = rabbitMQContainer.adminUsername, + password = rabbitMQContainer.adminPassword, + ), + ConnectionManagerConfiguration( + subscriberName = "test", + prefetchCount = prefetchCount, + confirmationTimeout = confirmationTimeout, + ), + ) { + LOGGER.error { "Fatal connection problem" } + } + + companion object { + private const val DEFAULT_PREFETCH_COUNT = 10 + private val DEFAULT_CONFIRMATION_TIMEOUT: Duration = Duration.ofSeconds(1) + } } \ No newline at end of file From e049907cf65a174dbe8d066258adbb2778f75ca0 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Fri, 18 Nov 2022 16:11:03 +0400 Subject: [PATCH 079/154] [TH2-4443] Replaced CopyOnWriteArrayList to the concurrent hash set in the AbstractRabbitSubscriber class --- .../impl/rabbitmq/AbstractRabbitSubscriber.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index b885545cf..5234adf0b 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -26,21 +26,20 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import com.google.protobuf.Message; import com.rabbitmq.client.Delivery; - import io.prometheus.client.Counter; import io.prometheus.client.Histogram; import io.prometheus.client.Histogram.Timer; - import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Objects; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; @@ -73,7 +72,7 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber> listeners = new CopyOnWriteArrayList<>(); + private final Set> listeners = ConcurrentHashMap.newKeySet(); private final String queue; private final ConnectionManager connectionManager; private final AtomicReference consumerMonitor = new AtomicReference<>(); @@ -125,7 +124,7 @@ public void addListener(ConfirmationMessageListener messageListener) { + " because only one listener with manual confirmation is allowed per queue"); } } - if(listeners.addIfAbsent(messageListener)) { + if(listeners.add(messageListener)) { hasManualSubscriber |= isManual; subscribe(); } @@ -157,7 +156,7 @@ public void close() throws IOException { monitor.unsubscribe(); } - ListIterator> listIterator = listeners.listIterator(); + Iterator> listIterator = listeners.iterator(); while (listIterator.hasNext()) { ConfirmationMessageListener listener = listIterator.next(); listIterator.remove(); From 243bfc5b2de1cc9735cef4e6be8759a98dee8932 Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Mon, 21 Nov 2022 11:02:40 +0400 Subject: [PATCH 080/154] Apply suggestions from code review Co-authored-by: Ivan Druzhinin <46186068+ivandruzhinin@users.noreply.github.com> --- .../exactpro/th2/common/schema/message/MessageRouter.java | 2 +- .../message/impl/rabbitmq/AbstractRabbitSubscriber.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index 2f9882374..c4d299ffb 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -54,7 +54,7 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< void init(@NotNull MessageRouterContext context); /** - * Create new exclusive queue and subscribe to it. Only own connection can use the particular queue. + * Create new exclusive queue and subscribe to it. Only declaring connection can use this queue. * Please note Exclusive queues are deleted when their declaring connection is closed or gone (e.g. due to underlying TCP connection loss). * They therefore are only suitable for client-specific transient state. * @return {@link SubscriberExclusiveMonitor} object to manage subscription. diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index 5234adf0b..5c86b6788 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -118,8 +118,7 @@ public void addListener(ConfirmationMessageListener messageListener) { try { publicLock.lock(); boolean isManual = ConfirmationMessageListener.isManual(messageListener); - if (isManual) { - if (hasManualSubscriber) { + if (isManual && hasManualSubscriber) { throw new IllegalStateException("cannot subscribe listener " + messageListener + " because only one listener with manual confirmation is allowed per queue"); } @@ -139,7 +138,7 @@ public void removeListener(ConfirmationMessageListener messageListener) { publicLock.lock(); boolean isManual = ConfirmationMessageListener.isManual(messageListener); if (listeners.remove(messageListener)) { - hasManualSubscriber = !(hasManualSubscriber && isManual); + hasManualSubscriber &= !isManual; messageListener.onClose(); } } finally { From 633a7de420e84dda35edfc1525df87698accd831 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 21 Nov 2022 11:04:15 +0400 Subject: [PATCH 081/154] [TH2-4262] Corrected classes after review --- src/jmh/kotlin/MetricsUtilsBenchmark.kt | 2 -- .../exactpro/th2/common/schema/message/MessageRouter.java | 4 ++-- .../message/impl/rabbitmq/AbstractRabbitSubscriber.java | 6 +++--- .../message/impl/rabbitmq/connection/ConnectionManager.java | 6 +++--- ...berExclusiveMonitor.kt => ExclusiveSubscriberMonitor.kt} | 2 +- .../impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt | 4 ++-- .../schema/message/impl/rabbitmq/AbstractRabbitRouter.kt | 4 ++-- .../rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt | 4 ++-- 8 files changed, 15 insertions(+), 17 deletions(-) rename src/main/kotlin/com/exactpro/th2/common/schema/message/{SubscriberExclusiveMonitor.kt => ExclusiveSubscriberMonitor.kt} (92%) diff --git a/src/jmh/kotlin/MetricsUtilsBenchmark.kt b/src/jmh/kotlin/MetricsUtilsBenchmark.kt index df8b07126..31d5150bd 100644 --- a/src/jmh/kotlin/MetricsUtilsBenchmark.kt +++ b/src/jmh/kotlin/MetricsUtilsBenchmark.kt @@ -24,8 +24,6 @@ import com.exactpro.th2.common.message.direction import com.exactpro.th2.common.message.plusAssign import com.exactpro.th2.common.message.sequence import com.exactpro.th2.common.message.sessionAlias -import com.exactpro.th2.common.grpc.MessageGroup -import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.metrics.DIRECTION_LABEL import com.exactpro.th2.common.metrics.MESSAGE_TYPE_LABEL import com.exactpro.th2.common.metrics.SESSION_ALIAS_LABEL diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index 2f9882374..20b78253e 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -57,9 +57,9 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< * Create new exclusive queue and subscribe to it. Only own connection can use the particular queue. * Please note Exclusive queues are deleted when their declaring connection is closed or gone (e.g. due to underlying TCP connection loss). * They therefore are only suitable for client-specific transient state. - * @return {@link SubscriberExclusiveMonitor} object to manage subscription. + * @return {@link ExclusiveSubscriberMonitor} object to manage subscription. */ - SubscriberExclusiveMonitor subscribeExclusive(MessageListener callback); + ExclusiveSubscriberMonitor subscribeExclusive(MessageListener callback); /** * Listen ONE RabbitMQ queue by intersection schemas queues attributes diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index 5234adf0b..ba015a39b 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -115,8 +115,8 @@ public void start() throws Exception { @Override public void addListener(ConfirmationMessageListener messageListener) { + publicLock.lock(); try { - publicLock.lock(); boolean isManual = ConfirmationMessageListener.isManual(messageListener); if (isManual) { if (hasManualSubscriber) { @@ -135,8 +135,8 @@ public void addListener(ConfirmationMessageListener messageListener) { @Override public void removeListener(ConfirmationMessageListener messageListener) { + publicLock.lock(); try { - publicLock.lock(); boolean isManual = ConfirmationMessageListener.isManual(messageListener); if (listeners.remove(messageListener)) { hasManualSubscriber = !(hasManualSubscriber && isManual); @@ -149,8 +149,8 @@ public void removeListener(ConfirmationMessageListener messageListener) { @Override public void close() throws IOException { + publicLock.lock(); try { - publicLock.lock(); SubscriberMonitor monitor = consumerMonitor.getAndSet(null); if (monitor != null) { monitor.unsubscribe(); diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java index 8bc55b634..3190617d5 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java @@ -35,7 +35,7 @@ import javax.annotation.concurrent.GuardedBy; -import com.exactpro.th2.common.schema.message.SubscriberExclusiveMonitor; +import com.exactpro.th2.common.schema.message.ExclusiveSubscriberMonitor; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -301,7 +301,7 @@ public String queueDeclare() throws IOException { }); } - public SubscriberExclusiveMonitor basicConsume(String queue, ManualAckDeliveryCallback deliverCallback, CancelCallback cancelCallback) throws IOException { + public ExclusiveSubscriberMonitor basicConsume(String queue, ManualAckDeliveryCallback deliverCallback, CancelCallback cancelCallback) throws IOException { ChannelHolder holder = getChannelFor(PinId.forQueue(queue)); String tag = holder.mapWithLock(channel -> channel.basicConsume(queue, false, subscriberName + "_" + nextSubscriberId.getAndIncrement(), (tagTmp, delivery) -> { @@ -438,7 +438,7 @@ private static void basicAck(Channel channel, long deliveryTag) throws IOExcepti channel.basicAck(deliveryTag, false); } - private static class RabbitMqSubscriberMonitor implements SubscriberExclusiveMonitor { + private static class RabbitMqSubscriberMonitor implements ExclusiveSubscriberMonitor { private final ChannelHolder holder; private final String queue; diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/SubscriberExclusiveMonitor.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/ExclusiveSubscriberMonitor.kt similarity index 92% rename from src/main/kotlin/com/exactpro/th2/common/schema/message/SubscriberExclusiveMonitor.kt rename to src/main/kotlin/com/exactpro/th2/common/schema/message/ExclusiveSubscriberMonitor.kt index bd05d211e..2350bc91a 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/SubscriberExclusiveMonitor.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/ExclusiveSubscriberMonitor.kt @@ -16,6 +16,6 @@ package com.exactpro.th2.common.schema.message -interface SubscriberExclusiveMonitor : SubscriberMonitor { +interface ExclusiveSubscriberMonitor : SubscriberMonitor { val queue: String } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index 2d6e88785..e53a3d779 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -19,7 +19,7 @@ import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.MessageRouterContext -import com.exactpro.th2.common.schema.message.SubscriberExclusiveMonitor +import com.exactpro.th2.common.schema.message.ExclusiveSubscriberMonitor import com.exactpro.th2.common.schema.message.SubscriberMonitor import com.exactpro.th2.common.schema.message.appendAttributes @@ -40,7 +40,7 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { this.groupBatchRouter = groupBatchRouter } - override fun subscribeExclusive(callback: MessageListener): SubscriberExclusiveMonitor { + override fun subscribeExclusive(callback: MessageListener): ExclusiveSubscriberMonitor { return groupBatchRouter.subscribeExclusive { consumerTag: String, message: MessageGroupBatch -> callback.handler(consumerTag, buildFromGroupBatch(message)) } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index 34b0c07ea..42306613b 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -89,7 +89,7 @@ abstract class AbstractRabbitRouter : MessageRouter { } } - override fun subscribeExclusive(callback: MessageListener): SubscriberExclusiveMonitor { + override fun subscribeExclusive(callback: MessageListener): ExclusiveSubscriberMonitor { val queue = connectionManager.queueDeclare() val pinConfig = PinConfiguration("", queue, "", isReadable = true, isWritable = false) @@ -98,7 +98,7 @@ abstract class AbstractRabbitRouter : MessageRouter { addListener(messageListener) } - return object: SubscriberExclusiveMonitor { + return object: ExclusiveSubscriberMonitor { override val queue: String = queue override fun unsubscribe() { diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt index 6fc9fb667..2a4921c7d 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt @@ -22,7 +22,7 @@ import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.plusAssign import com.exactpro.th2.common.schema.message.MessageRouter -import com.exactpro.th2.common.schema.message.SubscriberExclusiveMonitor +import com.exactpro.th2.common.schema.message.ExclusiveSubscriberMonitor import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration @@ -45,7 +45,7 @@ import org.mockito.kotlin.verify class TestRabbitMessageGroupBatchRouter { private val connectionConfiguration = ConnectionManagerConfiguration() - private val monitor: SubscriberExclusiveMonitor = mock { } + private val monitor: ExclusiveSubscriberMonitor = mock { } private val connectionManager: ConnectionManager = mock { on { configuration }.thenReturn(connectionConfiguration) on { basicConsume(any(), any(), any()) }.thenReturn(monitor) From 5285153b7918381aa6563b09d963a68b4c03b80c Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 21 Nov 2022 18:21:53 +0400 Subject: [PATCH 082/154] [TH2-4262] Updated grpc-common to 3.12.0 * consider message group batch metadata in common * kubernetes mode downloads log4j2 properties * added IntegrationTestRabbitMessageGroupBatchRouter --- build.gradle | 2 +- .../common/schema/factory/CommonFactory.java | 6 +- .../schema/message/MessageRouterUtils.kt | 17 ++- .../group/RabbitMessageGroupBatchRouter.kt | 5 +- .../RabbitMessageGroupBatchSubscriber.kt | 3 +- ...rationTestRabbitMessageGroupBatchRouter.kt | 103 ++++++++++++++++++ 6 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt diff --git a/build.gradle b/build.gradle index 36b865720..83a4cca5b 100644 --- a/build.gradle +++ b/build.gradle @@ -173,7 +173,7 @@ dependencies { api ("com.exactpro.th2:cradle-core:${cradleVersion}") { exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability } - api 'com.exactpro.th2:grpc-common:3.11.1' + api 'com.exactpro.th2:grpc-common:3.12.0-TH2-4262-reduce-load-on-a-separate-boxes-in-crawler-schema-3480850604-SNAPSHOT' jmh 'org.openjdk.jmh:jmh-core:0.9' jmh 'org.openjdk.jmh:jmh-generator-annprocess:0.9' diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index 399eb21f8..9d7645b9c 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -454,8 +454,8 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam Map boxData = boxConfigMap.getData(); Map rabbitMqData = rabbitMqConfigMap.getData(); Map cradleConfigData = cradleConfigMap.getData(); - @Nullable String loggingData = boxData.getOrDefault(LOG4J_PROPERTIES_NAME, - loggingConfigMap == null ? null : loggingConfigMap.getData().get(LOG4J_PROPERTIES_NAME) + @Nullable String loggingData = boxData.getOrDefault(LOG4J2_PROPERTIES_NAME, + loggingConfigMap == null ? null : loggingConfigMap.getData().get(LOG4J2_PROPERTIES_NAME) ); File generatedConfigsDirFile = configPath.toFile(); @@ -471,7 +471,7 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam box.setBoxName(boxName); if (loggingData != null) { - writeFile(configPath.resolve(LOG4J_PROPERTIES_NAME), loggingData); + writeFile(configPath.resolve(LOG4J2_PROPERTIES_NAME), loggingData); configureLogger(configPath.toString()); } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index 7afd77a54..0335d77d7 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -25,6 +25,8 @@ import com.exactpro.th2.common.event.EventUtils import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.EventBatch import com.exactpro.th2.common.grpc.MessageGroupBatch +import com.exactpro.th2.common.grpc.MessageGroupBatchMetadata +import com.exactpro.th2.common.grpc.MessageGroupBatchOrBuilder import com.exactpro.th2.common.message.logId import com.exactpro.th2.common.message.toJson import com.exactpro.th2.common.schema.message.QueueAttribute.EVENT @@ -77,8 +79,14 @@ fun appendAttributes( } fun MessageGroupBatch.toShortDebugString(): String = buildString { - append("MessageGroupBatch(ids = ") + append("MessageGroupBatch ") + val sourceMetadata = metadata + if (sourceMetadata !== MessageGroupBatchMetadata.getDefaultInstance()) { + append("external user queue = ${sourceMetadata.externalUserQueue} ") + } + + append("(ids = ") groupsList.asSequence() .flatMap { it.messagesList.asSequence() } .map(AnyMessage::logId) @@ -86,4 +94,11 @@ fun MessageGroupBatch.toShortDebugString(): String = buildString { .apply { append(this) } append(')') +} + +fun MessageGroupBatchOrBuilder.toBuilderWithMetadata(): MessageGroupBatch.Builder = MessageGroupBatch.newBuilder().apply { + val sourceMetadata = this@toBuilderWithMetadata.metadata + if (sourceMetadata !== MessageGroupBatchMetadata.getDefaultInstance()) { + metadata = sourceMetadata + } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt index d1baf2502..3303531c9 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt @@ -28,6 +28,7 @@ import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter +import com.exactpro.th2.common.schema.message.toBuilderWithMetadata import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName import com.google.protobuf.Message @@ -49,7 +50,7 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() return message } - val builder = MessageGroupBatch.newBuilder() + val builder = message.toBuilderWithMetadata() message.groupsList.forEach { group -> if (group.messagesList.all { filterMessage(it, pinConfiguration.filters) }) { builder.addGroups(group) @@ -82,7 +83,7 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() return RabbitMessageGroupBatchSubscriber( connectionManager, pinConfig.queue, - FilterFunction { msg: Message, filters: List -> filterMessage(msg, filters) }, + { msg: Message, filters: List -> filterMessage(msg, filters) }, pinName, pinConfig.filters, connectionManager.configuration.messageRecursionLimit diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt index 1d1b9be5e..c956464ef 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt @@ -28,6 +28,7 @@ import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback +import com.exactpro.th2.common.schema.message.toBuilderWithMetadata import com.exactpro.th2.common.schema.message.impl.rabbitmq.group.RabbitMessageGroupBatchRouter.Companion.MESSAGE_GROUP_TYPE import com.exactpro.th2.common.schema.message.toShortDebugString import com.google.protobuf.CodedInputStream @@ -75,7 +76,7 @@ class RabbitMessageGroupBatchSubscriber( } .toList() - return if (groups.isEmpty()) null else MessageGroupBatch.newBuilder().addAllGroups(groups).build() + return if (groups.isEmpty()) null else batch.toBuilderWithMetadata().addAllGroups(groups).build() } override fun handle( diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt new file mode 100644 index 000000000..af9ea1799 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2022-2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.message.impl.rabbitmq.group + +import com.exactpro.th2.common.annotations.IntegrationTest +import com.exactpro.th2.common.grpc.MessageGroupBatch +import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration +import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import mu.KotlinLogging +import org.junit.jupiter.api.Test +import org.mockito.kotlin.mock +import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.test.assertTrue + +@IntegrationTest +class IntegrationTestRabbitMessageGroupBatchRouter { + + @Test + fun `subscribe to exclusive queue`() { + RabbitMQContainer(DockerImageName.parse("rabbitmq:3.8-management-alpine")) + .use { rabbitMQContainer -> + rabbitMQContainer.start() + LOGGER.info { "Started with port ${rabbitMQContainer.amqpPort}" } + + createConnectionManager(rabbitMQContainer).use { firstManager -> + createRouter(firstManager).use { firstRouter -> + createConnectionManager(rabbitMQContainer).use { secondManager -> + createRouter(secondManager).use { secondRouter -> + val counter = CountDownLatch(1) + val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } + try { + + secondRouter.sendExclusive(monitor.queue, MessageGroupBatch.getDefaultInstance()) + assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } + + } finally { + monitor.unsubscribe() + } + } + } + } + } + } + } + + private fun createRouter(connectionManager: ConnectionManager) = RabbitMessageGroupBatchRouter() + .apply { + init( + DefaultMessageRouterContext( + connectionManager, + mock { }, + MessageRouterConfiguration() + ) + ) + } + + private fun createConnectionManager( + rabbitMQContainer: RabbitMQContainer, + prefetchCount: Int = DEFAULT_PREFETCH_COUNT, + confirmationTimeout: Duration = DEFAULT_CONFIRMATION_TIMEOUT + ) = ConnectionManager( + RabbitMQConfiguration( + host = rabbitMQContainer.host, + vHost = "", + port = rabbitMQContainer.amqpPort, + username = rabbitMQContainer.adminUsername, + password = rabbitMQContainer.adminPassword, + ), + ConnectionManagerConfiguration( + subscriberName = "test", + prefetchCount = prefetchCount, + confirmationTimeout = confirmationTimeout, + ), + ) { + LOGGER.error { "Fatal connection problem" } + } + + companion object { + private val LOGGER = KotlinLogging.logger { } + + private const val DEFAULT_PREFETCH_COUNT = 10 + private val DEFAULT_CONFIRMATION_TIMEOUT: Duration = Duration.ofSeconds(1) + } +} \ No newline at end of file From 14f209679226f7f6c0d23950ad0960b1c785869b Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Thu, 24 Nov 2022 13:27:17 +0400 Subject: [PATCH 083/154] [TH2-4299] added subscribeWithManualAck to AbstractGroupBatchAdapterRouter --- .../AbstractGroupBatchAdapterRouter.kt | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt index 3ce3cef16..c2f72a575 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractGroupBatchAdapterRouter.kt @@ -17,6 +17,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.schema.message.DeliveryMetadata +import com.exactpro.th2.common.schema.message.ManualConfirmationListener import com.exactpro.th2.common.schema.message.MessageListener import com.exactpro.th2.common.schema.message.MessageRouter import com.exactpro.th2.common.schema.message.MessageRouterContext @@ -56,6 +57,35 @@ abstract class AbstractGroupBatchAdapterRouter : MessageRouter { ) } + override fun subscribeWithManualAck( + callback: ManualConfirmationListener, + vararg queueAttr: String + ): SubscriberMonitor { + val listener = + ManualConfirmationListener { deliveryMetadata, message, confirmation -> + callback.handle(deliveryMetadata, buildFromGroupBatch(message), confirmation) + } + + return groupBatchRouter.subscribeWithManualAck( + listener, + *appendAttributes(*queueAttr) { getRequiredSubscribeAttributes() }.toTypedArray() + ) + } + + override fun subscribeAllWithManualAck( + callback: ManualConfirmationListener, + vararg queueAttr: String + ): SubscriberMonitor { + val listener = + ManualConfirmationListener { deliveryMetadata, message, confirmation -> + callback.handle(deliveryMetadata, buildFromGroupBatch(message), confirmation) + } + + return groupBatchRouter.subscribeAllWithManualAck(listener, + *appendAttributes(*queueAttr) { getRequiredSubscribeAttributes() }.toTypedArray() + ) + } + override fun send(messageBatch: T, vararg attributes: String) { groupBatchRouter.send( buildGroupBatch(messageBatch), From 7b68c40984aef6b4a89aecf45110f8421e148c10 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 29 Nov 2022 16:12:05 +0400 Subject: [PATCH 084/154] [th2-4262] Added tests for serialize message batch metadata --- ...rationTestRabbitMessageGroupBatchRouter.kt | 43 +++++++++++++++- .../TestRabbitMessageGroupBatchRouter.kt | 49 +++++++++++++++++-- src/test/resources/log4j2.properties | 40 +++++++++++++++ 3 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/log4j2.properties diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt index af9ea1799..93b603731 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt @@ -21,6 +21,7 @@ import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterC import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.rabbitmq.client.BuiltinExchangeType import mu.KotlinLogging import org.junit.jupiter.api.Test import org.mockito.kotlin.mock @@ -36,7 +37,7 @@ class IntegrationTestRabbitMessageGroupBatchRouter { @Test fun `subscribe to exclusive queue`() { - RabbitMQContainer(DockerImageName.parse("rabbitmq:3.8-management-alpine")) + RabbitMQContainer(DockerImageName.parse(RABBITMQ_3_8_MANAGEMENT_ALPINE)) .use { rabbitMQContainer -> rabbitMQContainer.start() LOGGER.info { "Started with port ${rabbitMQContainer.amqpPort}" } @@ -62,6 +63,41 @@ class IntegrationTestRabbitMessageGroupBatchRouter { } } + @Test + fun `send receive message group batch`() { + RabbitMQContainer(DockerImageName.parse(RABBITMQ_3_8_MANAGEMENT_ALPINE)) + .withExchange(EXCHANGE, BuiltinExchangeType.DIRECT.type, false, false, true, emptyMap()) + .withQueue(QUEUE_NAME) + .withBinding(EXCHANGE, QUEUE_NAME, emptyMap(), ROUTING_KEY, "queue") + .use { rabbitMQContainer -> + rabbitMQContainer.start() + LOGGER.info { "Started with port ${rabbitMQContainer.amqpPort}" } + + createConnectionManager(rabbitMQContainer).use { firstManager -> + createRouter(firstManager).use { firstRouter -> + createConnectionManager(rabbitMQContainer).use { secondManager -> + createRouter(secondManager).use { secondRouter -> + firstRouter.subscribe({ _, _ -> + + }) + + val counter = CountDownLatch(1) + val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } + try { + + secondRouter.sendExclusive(monitor.queue, MessageGroupBatch.getDefaultInstance()) + assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } + + } finally { + monitor.unsubscribe() + } + } + } + } + } + } + } + private fun createRouter(connectionManager: ConnectionManager) = RabbitMessageGroupBatchRouter() .apply { init( @@ -97,6 +133,11 @@ class IntegrationTestRabbitMessageGroupBatchRouter { companion object { private val LOGGER = KotlinLogging.logger { } + private const val RABBITMQ_3_8_MANAGEMENT_ALPINE = "rabbitmq:3.8-management-alpine" + private const val ROUTING_KEY = "routingKey" + private const val QUEUE_NAME = "queue" + private const val EXCHANGE = "test-exchange" + private const val DEFAULT_PREFETCH_COUNT = 10 private val DEFAULT_CONFIRMATION_TIMEOUT: Duration = Duration.ofSeconds(1) } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt index 2a4921c7d..9ad719ca8 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt @@ -32,6 +32,7 @@ import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterC import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.function.Executable @@ -96,6 +97,27 @@ class TestRabbitMessageGroupBatchRouter { ) )) + @Test + fun `publishes message group batch with metadata`() { + val batch = MessageGroupBatch.newBuilder().apply { + metadataBuilder.apply { + externalUserQueue = "externalUserQueue" + } + addGroupsBuilder().apply { + this += message("test-message", Direction.FIRST, "test-alias") + } + }.build() + + router.send(batch, "test") + + val captor = argumentCaptor() + verify(connectionManager).basicPublish(eq("test-exchange"), eq("test2"), anyOrNull(), captor.capture()) + val publishedBytes = captor.firstValue + assertArrayEquals(batch.toByteArray(), publishedBytes) { + "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" + } + } + @Test fun `does not publish anything if all messages are filtered`() { router.send( @@ -119,7 +141,7 @@ class TestRabbitMessageGroupBatchRouter { val captor = argumentCaptor() verify(connectionManager).basicPublish(eq("test-exchange"), eq("test2"), anyOrNull(), captor.capture()) val publishedBytes = captor.firstValue - Assertions.assertArrayEquals(batch.toByteArray(), publishedBytes) { + assertArrayEquals(batch.toByteArray(), publishedBytes) { "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" } } @@ -171,13 +193,13 @@ class TestRabbitMessageGroupBatchRouter { Assertions.assertAll( Executable { val publishedBytes = captor.firstValue - Assertions.assertArrayEquals(originalBytes, publishedBytes) { + assertArrayEquals(originalBytes, publishedBytes) { "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" } }, Executable { val publishedBytes = captor.secondValue - Assertions.assertArrayEquals(originalBytes, publishedBytes) { + assertArrayEquals(originalBytes, publishedBytes) { "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" } } @@ -210,6 +232,27 @@ class TestRabbitMessageGroupBatchRouter { ) ) + @Test + fun `publishes message group batch with metadata`() { + val batch = MessageGroupBatch.newBuilder().apply { + metadataBuilder.apply { + externalUserQueue = "externalUserQueue" + } + addGroupsBuilder().apply { + this += message("test-message", Direction.FIRST, "test-alias") + } + }.build() + + router.send(batch, "test") + + val captor = argumentCaptor() + verify(connectionManager).basicPublish(eq("test-exchange"), eq("test2"), anyOrNull(), captor.capture()) + val publishedBytes = captor.firstValue + assertArrayEquals(batch.toByteArray(), publishedBytes) { + "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" + } + } + @Test fun `subscribes to correct queue`() { val monitor = router.subscribe(mock { }, "1") diff --git a/src/test/resources/log4j2.properties b/src/test/resources/log4j2.properties new file mode 100644 index 000000000..db0ed801f --- /dev/null +++ b/src/test/resources/log4j2.properties @@ -0,0 +1,40 @@ +################################################################################ +# Copyright 2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +name = CommonJConfig +# Logging level related to initialization of Log4j +status = warn +# Package where to search plugins +packages = io.prometheus.client.log4j2 + +# Console appender configuration +appender.console.type = Console +appender.console.name = ConsoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n + +# Prometheus appender plugin configuration +appender.Prometheus.name = Prometheus +appender.Prometheus.type = Prometheus + +logger.prometheusLogger.name= prometheusLogger +logger.prometheusLogger.level= INFO +logger.prometheusLogger.appenderRef.ReportingAppender.ref= Prometheus + +# Root logger level +rootLogger.level = INFO +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = ConsoleLogger + From 05edfd5c5c160a3aa04d1fbd43e007264965be88 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Thu, 22 Dec 2022 12:13:06 +0400 Subject: [PATCH 085/154] [th2-4262] changes by PR comments --- .../th2/common/metrics}/MetricsUtilsBenchmark.kt | 9 +-------- .../common/schema/grpc/router/AbstractGrpcRouter.java | 6 ++++++ .../th2/common/schema/message/MessageRouter.java | 4 ++-- .../impl/rabbitmq/connection/ConnectionManager.java | 7 ++++++- .../th2/common/grpc/router/AbstractGrpcInterceptor.kt | 8 ++++++-- .../th2/common/schema/message/MessageRouterUtils.kt | 11 ++++------- 6 files changed, 25 insertions(+), 20 deletions(-) rename src/jmh/kotlin/{ => com/exactpro/th2/common/metrics}/MetricsUtilsBenchmark.kt (93%) diff --git a/src/jmh/kotlin/MetricsUtilsBenchmark.kt b/src/jmh/kotlin/com/exactpro/th2/common/metrics/MetricsUtilsBenchmark.kt similarity index 93% rename from src/jmh/kotlin/MetricsUtilsBenchmark.kt rename to src/jmh/kotlin/com/exactpro/th2/common/metrics/MetricsUtilsBenchmark.kt index 31d5150bd..02a94b48c 100644 --- a/src/jmh/kotlin/MetricsUtilsBenchmark.kt +++ b/src/jmh/kotlin/com/exactpro/th2/common/metrics/MetricsUtilsBenchmark.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.exactpro.th2 +package com.exactpro.th2.common.metrics import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.Direction @@ -24,13 +24,6 @@ import com.exactpro.th2.common.message.direction import com.exactpro.th2.common.message.plusAssign import com.exactpro.th2.common.message.sequence import com.exactpro.th2.common.message.sessionAlias -import com.exactpro.th2.common.metrics.DIRECTION_LABEL -import com.exactpro.th2.common.metrics.MESSAGE_TYPE_LABEL -import com.exactpro.th2.common.metrics.SESSION_ALIAS_LABEL -import com.exactpro.th2.common.metrics.TH2_PIN_LABEL -import com.exactpro.th2.common.metrics.incrementDroppedMetrics -import com.exactpro.th2.common.metrics.incrementTotalMetrics -import com.exactpro.th2.common.metrics.incrementTotalMetricsOld import io.prometheus.client.Counter import io.prometheus.client.Gauge import org.openjdk.jmh.annotations.Benchmark diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java index 29bd45255..739ea0359 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java @@ -68,6 +68,12 @@ public abstract class AbstractGrpcRouter implements GrpcRouter { protected GrpcRouterConfiguration routerConfiguration; protected GrpcConfiguration configuration; + // Metrics below are collectd to maps to improve getting .Child performance. + // Prometheus metric has the `labels` tread-safe method for getting .Child but its logic includes: + // * array String creation + // * covert the array to list + // * search .Child by the list + protected static final Counter GRPC_INVOKE_CALL_TOTAL = Counter.build() .name("th2_grpc_invoke_call_total") .labelNames(CommonMetrics.TH2_PIN_LABEL, CommonMetrics.GRPC_SERVICE_NAME_LABEL, CommonMetrics.GRPC_METHOD_NAME_LABEL) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index 21d89cdda..60175b9e3 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -53,9 +53,9 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< void init(@NotNull MessageRouterContext context); /** - * Create new exclusive queue and subscribe to it. Only declaring connection can use this queue. + * Creates a new exclusive queue and subscribes to it. Only declaring connection can use this queue. * Please note Exclusive queues are deleted when their declaring connection is closed or gone (e.g. due to underlying TCP connection loss). - * They therefore are only suitable for client-specific transient state. + * They, therefore, are only suitable for client-specific transient states. * @return {@link ExclusiveSubscriberMonitor} object to manage subscription. */ ExclusiveSubscriberMonitor subscribeExclusive(MessageListener callback); diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java index 3c6603709..08e7cb888 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java @@ -293,7 +293,12 @@ public void basicPublish(String exchange, String routingKey, BasicProperties pro public String queueDeclare() throws IOException { ChannelHolder holder = new ChannelHolder(this::createChannel, this::waitForConnectionRecovery, configuration.getPrefetchCount()); return holder.mapWithLock( channel -> { - String queue = holder.channel.queueDeclare("", false, true, false, Collections.emptyMap()).getQueue(); + String queue = channel.queueDeclare( + "", // queue name + false, // durable + true, // exclusive + false, // autoDelete + Collections.emptyMap()).getQueue(); LOGGER.info("Declared exclusive '{}' queue", queue); putChannelFor(PinId.forQueue(queue), holder); return queue; diff --git a/src/main/kotlin/com/exactpro/th2/common/grpc/router/AbstractGrpcInterceptor.kt b/src/main/kotlin/com/exactpro/th2/common/grpc/router/AbstractGrpcInterceptor.kt index 167c065a5..95b08ac21 100644 --- a/src/main/kotlin/com/exactpro/th2/common/grpc/router/AbstractGrpcInterceptor.kt +++ b/src/main/kotlin/com/exactpro/th2/common/grpc/router/AbstractGrpcInterceptor.kt @@ -15,7 +15,7 @@ package com.exactpro.th2.common.grpc.router -import com.exactpro.th2.common.value.toValue +import com.google.protobuf.Message import io.grpc.MethodDescriptor import io.prometheus.client.Counter import mu.KotlinLogging @@ -39,8 +39,12 @@ abstract class AbstractGrpcInterceptor ( sizeBytesCounter: Counter.Child?, methodInvokeCounter: Counter.Child, ) { + require(this is Message) { + "Passed object of ${this::class.java} class is not implement the ${Message::class.java} interface" + } + val size: Int? = sizeBytesCounter?.let { - toValue().serializedSize.also { + serializedSize.also { sizeBytesCounter.inc(it.toDouble()) } } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index 504ca3767..7a4f7f49d 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -26,7 +26,6 @@ import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.EventBatch import com.exactpro.th2.common.grpc.EventID import com.exactpro.th2.common.grpc.MessageGroupBatch -import com.exactpro.th2.common.grpc.MessageGroupBatchMetadata import com.exactpro.th2.common.grpc.MessageGroupBatchOrBuilder import com.exactpro.th2.common.message.logId import com.exactpro.th2.common.message.toJson @@ -96,9 +95,8 @@ fun appendAttributes( fun MessageGroupBatch.toShortDebugString(): String = buildString { append("MessageGroupBatch ") - val sourceMetadata = metadata - if (sourceMetadata !== MessageGroupBatchMetadata.getDefaultInstance()) { - append("external user queue = ${sourceMetadata.externalQueue} ") + if (hasMetadata()) { + append("external user queue = ${metadata.externalQueue} ") } append("(ids = ") @@ -112,8 +110,7 @@ fun MessageGroupBatch.toShortDebugString(): String = buildString { } fun MessageGroupBatchOrBuilder.toBuilderWithMetadata(): MessageGroupBatch.Builder = MessageGroupBatch.newBuilder().apply { - val sourceMetadata = this@toBuilderWithMetadata.metadata - if (sourceMetadata !== MessageGroupBatchMetadata.getDefaultInstance()) { - metadata = sourceMetadata + if (this@toBuilderWithMetadata.hasMetadata()) { + metadata = this@toBuilderWithMetadata.metadata } } \ No newline at end of file From b4dbd54e78a70812e074252c6d00fa721542f020 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 4 Jan 2023 13:16:07 +0400 Subject: [PATCH 086/154] [th2-4560] Added messageId validation into Event class --- .../com/exactpro/th2/common/event/Event.java | 19 ++++++++--- .../th2/common/message/MessageFilterUtils.kt | 6 +++- .../th2/common/message/MessageUtils.kt | 32 ++++++++++++------- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/event/Event.java b/src/main/java/com/exactpro/th2/common/event/Event.java index 73617d0a0..4e9d91068 100644 --- a/src/main/java/com/exactpro/th2/common/event/Event.java +++ b/src/main/java/com/exactpro/th2/common/event/Event.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,8 +37,10 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import com.exactpro.th2.common.message.MessageUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -53,16 +55,18 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.ByteString; +//TODO: move to common-utils-j +@SuppressWarnings("unused") public class Event { private static final Logger LOGGER = LoggerFactory.getLogger(Event.class); - public static final String UNKNOWN_EVENT_NAME = "Unknown event name"; public static final String UNKNOWN_EVENT_TYPE = "Unknown event type"; public static final EventID DEFAULT_EVENT_ID = EventID.getDefaultInstance(); - protected static final ThreadLocal OBJECT_MAPPER = ThreadLocal.withInitial(() -> new ObjectMapper().setSerializationInclusion(NON_NULL)); + protected final String UUID = generateUUID(); + protected final AtomicLong ID_COUNTER = new AtomicLong(); - protected final String id = generateUUID(); + protected final String id = UUID + '-' + ID_COUNTER.incrementAndGet(); protected final List subEvents = new ArrayList<>(); protected final List attachedMessageIDS = new ArrayList<>(); protected final List body = new ArrayList<>(); @@ -231,7 +235,12 @@ public Event exception(@NotNull Throwable throwable, boolean includeCauses) { * @return current event */ public Event messageID(MessageID attachedMessageID) { - attachedMessageIDS.add(requireNonNull(attachedMessageID, "Attached message id can't be null")); + requireNonNull(attachedMessageID, "Attached message id can't be null"); + if (MessageUtils.isValid(attachedMessageID)) { + attachedMessageIDS.add(attachedMessageID); + } else { + throw new IllegalArgumentException("Attached " + MessageUtils.toJson(attachedMessageID) + " message id"); + } return this; } diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt index 88b0ef747..d0813db31 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt @@ -41,6 +41,7 @@ import com.exactpro.th2.common.grpc.ValueFilter.KindCase.SIMPLE_FILTER import com.exactpro.th2.common.value.emptyValueFilter import com.exactpro.th2.common.value.toValueFilter import com.fasterxml.jackson.annotation.JsonProperty +import java.util.* private val DEFAULT_TIME_PRECISION_REGEX = Regex("(\\d[HMS])(?!\$)") @@ -127,7 +128,10 @@ private fun RootComparisonSettings.toTreeTableEntry(): TreeTableEntry = Collecti if (hasTimePrecision()) { val timePrecision = timePrecision.toJavaDuration().toString().substring(2) row("time-precision", RowBuilder() - .column(IgnoreFieldColumn(DEFAULT_TIME_PRECISION_REGEX.replace(timePrecision, "$1 ").toLowerCase())) + .column( + IgnoreFieldColumn( + DEFAULT_TIME_PRECISION_REGEX.replace(timePrecision, "$1 ").lowercase(Locale.getDefault()) + )) .build()) } if (decimalPrecision.isNotBlank()) { diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt index 3ea8c5ba7..2783474f7 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ @file:JvmName("MessageUtils") +@file:Suppress("unused") package com.exactpro.th2.common.message @@ -40,6 +41,7 @@ import com.exactpro.th2.common.grpc.Message import com.exactpro.th2.common.grpc.MessageFilter import com.exactpro.th2.common.grpc.MessageGroup import com.exactpro.th2.common.grpc.MessageID +import com.exactpro.th2.common.grpc.MessageIDOrBuilder import com.exactpro.th2.common.grpc.MessageMetadata import com.exactpro.th2.common.grpc.MessageOrBuilder import com.exactpro.th2.common.grpc.MetadataFilter @@ -78,9 +80,7 @@ import java.math.BigInteger import java.time.Instant import java.time.LocalDateTime import java.time.ZoneOffset -import java.util.Calendar -import java.util.Date -import java.util.TimeZone +import java.util.* typealias FieldValues = Map typealias FieldValueFilters = Map @@ -97,7 +97,7 @@ operator fun Message.Builder.get(key: String): Value? = getField(key) fun Message.Builder.getField(fieldName: String): Value? = getFieldsOrDefault(fieldName, null) fun Message.hasField(key: String) : Boolean = fieldsMap.containsKey(key) -fun Message.Builder.hasField(key: String) : Boolean = fieldsMap.containsKey(key); +fun Message.Builder.hasField(key: String) : Boolean = fieldsMap.containsKey(key) fun Message.getString(fieldName: String): String? = getField(fieldName)?.getString() fun Message.getInt(fieldName: String): Int? = getField(fieldName)?.getInt() @@ -127,9 +127,9 @@ fun Message.Builder.updateMessage(key: String, updateFunc: Message.Builder.() -> fun Message.Builder.updateString(key: String, updateFunc: String.() -> String) : Message.Builder = apply { updateField(key) { updateString(updateFunc) } } fun Message.Builder.updateOrAddField(key: String, updateFunc: (Value.Builder?) -> ValueOrBuilder?): Message.Builder = apply { set(key, updateFunc(getField(key)?.toBuilder())) } -fun Message.Builder.updateOrAddList(key: String, updateFunc: (ListValue.Builder?) -> ListValueOrBuilder) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddList(updateFunc) ?: updateFunc(null)?.toValue() } } -fun Message.Builder.updateOrAddMessage(key: String, updateFunc: (Message.Builder?) -> MessageOrBuilder) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddMessage(updateFunc) ?: updateFunc(null)?.toValue() } } -fun Message.Builder.updateOrAddString(key: String, updateFunc:(String?) -> String) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddString(updateFunc) ?: updateFunc(null)?.toValue() } } +fun Message.Builder.updateOrAddList(key: String, updateFunc: (ListValue.Builder?) -> ListValueOrBuilder) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddList(updateFunc) ?: updateFunc(null).toValue() } } +fun Message.Builder.updateOrAddMessage(key: String, updateFunc: (Message.Builder?) -> MessageOrBuilder) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddMessage(updateFunc) ?: updateFunc(null).toValue() } } +fun Message.Builder.updateOrAddString(key: String, updateFunc:(String?) -> String) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddString(updateFunc) ?: updateFunc(null).toValue() } } fun Message.Builder.addField(key: String, value: Any?): Message.Builder = apply { putFields(key, value?.toValue() ?: nullValue()) } @@ -388,10 +388,14 @@ var RawMessage.Builder.subsequence } val Message.logId: String - get() = "$sessionAlias:${direction.toString().toLowerCase()}:$sequence${subsequence.joinToString("") { ".$it" }}" + get() = "$sessionAlias:${ + direction.toString().lowercase(Locale.getDefault()) + }:$sequence${subsequence.joinToString("") { ".$it" }}" val RawMessage.logId: String - get() = "$sessionAlias:${direction.toString().toLowerCase()}:$sequence${subsequence.joinToString("") { ".$it" }}" + get() = "$sessionAlias:${ + direction.toString().lowercase(Locale.getDefault()) + }:$sequence${subsequence.joinToString("") { ".$it" }}" val AnyMessage.logId: String get() = when (kindCase) { @@ -425,7 +429,7 @@ val AnyMessage.bookName: BookName fun getDebugString(className: String, ids: List): String { val sessionAliasAndDirection = getSessionAliasAndDirection(ids[0]) val sequences = ids.joinToString { it.sequence.toString() } - return "$className: session_alias = ${sessionAliasAndDirection[0]}, direction = ${sessionAliasAndDirection[1]}, sequnces = $sequences" + return "$className: session_alias = ${sessionAliasAndDirection[0]}, direction = ${sessionAliasAndDirection[1]}, sequences = $sequences" } @JvmOverloads @@ -443,6 +447,12 @@ fun Message.toTreeTable(): TreeTable = TreeTableBuilder().apply { } }.build() +val MessageIDOrBuilder.isValid: Boolean + get() = bookName.isNotBlank() + && hasConnectionId() && connectionId.sessionAlias.isNotBlank() + && hasTimestamp() && timestamp.seconds > 0 && timestamp.nanos > 0 + && sequence > 0 + private fun Value.toTreeTableEntry(): TreeTableEntry = when { hasMessageValue() -> CollectionBuilder().apply { for ((key, value) in messageValue.fieldsMap) { From 088b4d6cbee1239b030b9c04eb3e1854538a827b Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 4 Jan 2023 17:35:09 +0400 Subject: [PATCH 087/154] [TH2-4561] Updated bom to 4.0.2 * Removed log4j 1.x from dependency --- README.md | 2 + build.gradle | 18 +++--- .../schema/factory/AbstractCommonFactory.java | 6 +- .../common/schema/factory/CommonFactory.java | 9 +-- .../common/schema/util/Log4jConfigUtils.kt | 58 ++++++------------- 5 files changed, 34 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 3cc101645..155c6864f 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,8 @@ dependencies { ### 4.0.0 + Migration to books/pages cradle 4.0.0 ++ Migration to bom 4.0.2 ++ Removed log4j 1.x from dependency + Removed `cradleInstanceName` parameter from `cradle.json` + Added `prepareStorage` property to `cradle.json` + `com.exactpro.th2.common.event.Event.toProto...()` by `parentEventId`/`bookName`/`(bookName + scope)` diff --git a/build.gradle b/build.gradle index 24703f9d2..7c14daa9f 100644 --- a/build.gradle +++ b/build.gradle @@ -166,7 +166,7 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform('com.exactpro.th2:bom:3.2.0') + api platform('com.exactpro.th2:bom:4.0.2') api "com.exactpro.th2:cradle-core:${cradleVersion}" api 'com.exactpro.th2:grpc-common:4.0.0-th2-2150-books-pages-2338290872-SNAPSHOT' @@ -178,16 +178,16 @@ dependencies { implementation "io.grpc:grpc-protobuf" implementation "io.grpc:grpc-core" implementation "io.grpc:grpc-netty" - implementation "io.grpc:grpc-services:1.32.1" //FIXME: add to bom + implementation "io.grpc:grpc-services" implementation "com.rabbitmq:amqp-client" implementation "org.jetbrains:annotations" implementation "org.apache.commons:commons-lang3" - implementation "org.apache.commons:commons-collections4:4.4" - implementation "org.apache.commons:commons-text:1.9" - implementation "commons-cli:commons-cli:1.5.0" + implementation "org.apache.commons:commons-collections4" + implementation "org.apache.commons:commons-text" + implementation "commons-cli:commons-cli" implementation "com.fasterxml.jackson.core:jackson-core" implementation "com.fasterxml.jackson.core:jackson-databind" @@ -197,21 +197,17 @@ dependencies { // TODO: need to update jackson kotlin module to get rid of kotlin-reflect 1.3.72 implementation "com.fasterxml.jackson.module:jackson-module-kotlin" - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}" - implementation 'com.fasterxml.uuid:java-uuid-generator:4.0.1' implementation 'io.github.microutils:kotlin-logging:2.1.21' - implementation 'org.apache.logging.log4j:log4j-slf4j-impl' - implementation 'org.apache.logging.log4j:log4j-1.2-api' - implementation 'org.apache.logging.log4j:log4j-api' + implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' + implementation 'org.apache.logging.log4j:log4j-core' implementation 'io.prometheus:simpleclient' implementation 'io.prometheus:simpleclient_hotspot' implementation 'io.prometheus:simpleclient_httpserver' implementation 'io.prometheus:simpleclient_log4j2' - implementation 'io.prometheus:simpleclient_log4j' implementation 'io.fabric8:kubernetes-client:4.13.0' diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 49d00c8bd..03d7d1ca3 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -113,8 +113,6 @@ public abstract class AbstractCommonFactory implements AutoCloseable { protected static final String LOG4J_PROPERTIES_DEFAULT_PATH_OLD = "/home/etc"; protected static final String LOG4J_PROPERTIES_DEFAULT_PATH = "/var/th2/config"; protected static final String LOG4J2_PROPERTIES_NAME = "log4j2.properties"; - protected static final String LOG4J_PROPERTIES_NAME = "log4j.properties"; - protected static final ObjectMapper MAPPER = new ObjectMapper(); static { @@ -832,7 +830,7 @@ protected static void configureLogger(String... paths) { listPath.add(LOG4J_PROPERTIES_DEFAULT_PATH_OLD); listPath.addAll(Arrays.asList(requireNonNull(paths, "Paths can't be null"))); Log4jConfigUtils log4jConfigUtils = new Log4jConfigUtils(); - log4jConfigUtils.configure(listPath, LOG4J_PROPERTIES_NAME, LOG4J2_PROPERTIES_NAME); + log4jConfigUtils.configure(listPath, LOG4J2_PROPERTIES_NAME); loggingManifests(); } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index 0cb9c0e2c..dd4556185 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -71,6 +71,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -455,8 +456,8 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam Map boxData = boxConfigMap.getData(); Map rabbitMqData = rabbitMqConfigMap.getData(); Map cradleConfigData = cradleConfigMap.getData(); - @Nullable String loggingData = boxData.getOrDefault(LOG4J_PROPERTIES_NAME, - loggingConfigMap == null ? null : loggingConfigMap.getData().get(LOG4J_PROPERTIES_NAME) + @Nullable String loggingData = boxData.getOrDefault(LOG4J2_PROPERTIES_NAME, + loggingConfigMap == null ? null : loggingConfigMap.getData().get(LOG4J2_PROPERTIES_NAME) ); File generatedConfigsDirFile = configPath.toFile(); @@ -472,7 +473,7 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam box.setBoxName(boxName); if (loggingData != null) { - writeFile(configPath.resolve(LOG4J_PROPERTIES_NAME), loggingData); + writeFile(configPath.resolve(LOG4J2_PROPERTIES_NAME), loggingData); configureLogger(configPath.toString()); } diff --git a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt index 3e5159055..6b867d4cb 100644 --- a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt +++ b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt @@ -16,8 +16,6 @@ package com.exactpro.th2.common.schema.util import org.apache.log4j.PropertyConfigurator -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.core.LoggerContext import org.slf4j.LoggerFactory import java.net.MalformedURLException import java.nio.file.Files @@ -25,53 +23,33 @@ import java.nio.file.Path class Log4jConfigUtils { - private val LOGGER = LoggerFactory.getLogger(Log4jConfigUtils::class.java) - fun configure( pathList: List, - firstVersionFileName: String, - secondVersionFileName: String, + versionFileName: String ) { - val configPathList: Pair? = pathList.asSequence() - .flatMap { - listOf( - Pair(Path.of(it, firstVersionFileName), 1), - Pair(Path.of(it, secondVersionFileName), 2) - ) - } - .filter { Files.exists(it.first) } - .sortedByDescending { it.second } + pathList.asSequence() + .map { Path.of(it, versionFileName) } + .filter(Files::exists) .firstOrNull() - - when (configPathList?.second) { - 1 -> configureFirstLog4j(configPathList.first) - 2 -> configureSecondLog4j(configPathList.first) - null -> { + ?.let { path -> + try { + LOGGER.info("Trying to apply logger config from {}. Expecting log4j syntax", path) + PropertyConfigurator.configure(path.toUri().toURL()) + LOGGER.info("Logger configuration from {} file is applied", path) + } catch (e: MalformedURLException) { + LOGGER.error(e.message, e) + } + } + ?: run { LOGGER.info( - "Neither of {} paths contains config files {}, {}. Use default configuration", + "Neither of {} paths contains config file {}. Use default configuration", pathList, - firstVersionFileName, - secondVersionFileName + versionFileName ) } - } - } - - private fun configureFirstLog4j(path: Path) { - try { - LOGGER.info("Trying to apply logger config from {}. Expecting log4j syntax", path) - PropertyConfigurator.configure(path.toUri().toURL()) - LOGGER.info("Logger configuration from {} file is applied", path) - } catch (e: MalformedURLException) { - e.printStackTrace() - } } - private fun configureSecondLog4j(path: Path) { - LOGGER.info("Trying to apply logger config from {}. Expecting log4j2 syntax", path) - val context = LogManager.getContext(false) as LoggerContext - context.configLocation = path.toUri() - LOGGER.info("Logger configuration from {} file is applied", path) + companion object { + private val LOGGER = LoggerFactory.getLogger(Log4jConfigUtils::class.java) } - } \ No newline at end of file From c30dc24420aeb19470671b878259ca12782b73ea Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 4 Jan 2023 17:40:24 +0400 Subject: [PATCH 088/154] [TH2-4561] download dictionaries into two directories for old and actual format --- .../common/schema/factory/CommonFactory.java | 86 +++++++++++-------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index dd4556185..b847f13ec 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -499,7 +499,7 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam writeFile(boxConfigurationPath, boxConfig); } - writeDictionaries(boxName, configPath, dictionaryTypePath, dictionaries, configMaps.list()); + writeDictionaries(dictionaryTypePath, dictionaryAliasPath, dictionaries, configMaps.list()); } return new CommonFactory(settings); @@ -635,51 +635,61 @@ private static Path writeFile(Path configPath, String fileName, Map dictionaries, ConfigMapList configMapList) throws IOException { + private static void writeDictionaries(Path oldDictionariesDir, Path dictionariesDir, Map dictionaries, ConfigMapList configMapList) throws IOException { + createDirectory(dictionariesDir); + for(ConfigMap configMap : configMapList.getItems()) { String configMapName = configMap.getMetadata().getName(); - if(configMapName.startsWith(boxName) && configMapName.endsWith("-dictionary")) { - configMap.getData().forEach((fileName, base64) -> { - try { - writeFile(oldDictionariesDir.resolve(fileName), base64); - } catch (IOException e) { - LOGGER.error("Can not write dictionary '{}' from config map with name '{}'", fileName, configMapName); - } - }); - } - } - for (Map.Entry entry : dictionaries.entrySet()) { - DictionaryType type = entry.getKey(); - String dictionaryName = entry.getValue(); - for (ConfigMap dictionaryConfigMap : configMapList.getItems()) { - String configName = dictionaryConfigMap.getMetadata().getName(); - if (configName.endsWith("-dictionary") && configName.substring(0, configName.lastIndexOf('-')).equals(dictionaryName)) { - Path dictionaryTypeDir = type.getDictionary(dictionariesDir); - - if (Files.notExists(dictionaryTypeDir)) { - Files.createDirectories(dictionaryTypeDir); - } else if (!Files.isDirectory(dictionaryTypeDir)) { - throw new IllegalStateException( - String.format("Can not save dictionary '%s' with type '%s', because '%s' is not directory", dictionaryName, type, dictionaryTypeDir) - ); - } + if (!configMapName.endsWith("-dictionary")) { + continue; + } - Set fileNameSet = dictionaryConfigMap.getData().keySet(); + String dictionaryName = configMapName.substring(0, configMapName.lastIndexOf('-')); - if (fileNameSet.size() != 1) { - throw new IllegalStateException( - String.format("Can not save dictionary '%s' with type '%s', because can not find dictionary data in config map", dictionaryName, type) - ); - } + dictionaries.entrySet().stream() + .filter(entry -> Objects.equals(entry.getValue(), dictionaryName)) + .forEach(entry -> { + DictionaryType type = entry.getKey(); + try { + Path dictionaryTypeDir = type.getDictionary(oldDictionariesDir); + createDirectory(dictionaryTypeDir); + + if (configMap.getData().size() != 1) { + throw new IllegalStateException( + String.format("Can not save dictionary '%s' with type '%s', because can not find dictionary data in config map", dictionaryName, type) + ); + } + + downloadFiles(dictionariesDir, configMap); + downloadFiles(dictionaryTypeDir, configMap); + } catch (Exception e) { + throw new IllegalStateException("Loading the " + dictionaryName + " dictionary with type " + type + " failures", e); + } + }); + } + } - String fileName = fileNameSet.stream().findFirst().orElse(null); - Path dictionaryPath = dictionaryTypeDir.resolve(fileName); - writeFile(dictionaryPath, dictionaryConfigMap.getData().get(fileName)); - LOGGER.debug("Dictionary written in folder: " + dictionaryPath); - break; + private static void downloadFiles(Path baseDir, ConfigMap configMap) { + String configMapName = configMap.getMetadata().getName(); + configMap.getData().forEach((fileName, base64) -> { + try { + Path path = baseDir.resolve(fileName); + if (!Files.exists(path)) { + writeFile(path, base64); + LOGGER.info("The '{}' config has been downloaded from the '{}' config map to the '{}' path", fileName, configMapName, path); } + } catch (IOException e) { + LOGGER.error("Can not download the '{}' file from the '{}' config map", fileName, configMapName); } + }); + } + + private static void createDirectory(Path dir) throws IOException { + if (Files.notExists(dir)) { + Files.createDirectories(dir); + } else if (!Files.isDirectory(dir)) { + throw new IllegalStateException("Can not save dictionary '" + dir + "' because the '" + dir + "' has already exist and isn't a directory"); } } From a4bb950c343f8d4014c4400c4d7535919bb89936 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Thu, 5 Jan 2023 13:14:47 +0400 Subject: [PATCH 089/154] [TH2-4262] Remove redundant code --- build.gradle | 3 +++ .../com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 88d4232a5..9a23595b7 100644 --- a/build.gradle +++ b/build.gradle @@ -222,6 +222,9 @@ dependencies { implementation 'io.prometheus:simpleclient_httpserver' implementation 'io.prometheus:simpleclient_log4j2' + implementation ('com.squareup.okhttp3:okhttp:4.10.0') { + because ('fix vulnerability in transitive dependency ') + } implementation 'io.fabric8:kubernetes-client:6.1.1' testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" diff --git a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt index fe21658e7..113b6e870 100644 --- a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt +++ b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt @@ -34,14 +34,10 @@ class Log4jConfigUtils { ?.let { path -> try { LOGGER.info("Trying to apply logger config from {}. Expecting log4j syntax", path) - LOGGER.debug("test") - LOGGER.trace("test") val loggerContext = LoggerContext.getContext(false) loggerContext.configLocation = path.toUri() loggerContext.reconfigure() LOGGER.info("Logger configuration from {} file is applied", path) - LOGGER.debug("test") - LOGGER.trace("test") } catch (e: MalformedURLException) { LOGGER.error(e.message, e) } From 183cf785eeb6a8e28d5aa6700eb6a3e7cbec164c Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 9 Jan 2023 10:54:44 +0400 Subject: [PATCH 090/154] [TH2-4262] Use box name as default scope for root event --- src/main/java/com/exactpro/th2/common/event/Event.java | 4 ++-- .../th2/common/schema/factory/AbstractCommonFactory.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/event/Event.java b/src/main/java/com/exactpro/th2/common/event/Event.java index be41ba53c..b69b5a057 100644 --- a/src/main/java/com/exactpro/th2/common/event/Event.java +++ b/src/main/java/com/exactpro/th2/common/event/Event.java @@ -308,12 +308,12 @@ public com.exactpro.th2.common.grpc.Event toProto(@NotNull String bookName) thro public com.exactpro.th2.common.grpc.Event toProto( @NotNull String bookName, - @NotNull String scope + @Nullable String scope ) throws JsonProcessingException { return toProto( null, requireNonBlankBookName(bookName), - requireNonBlankScope(scope) + scope ); } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 03d7d1ca3..13e9f13ed 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -634,7 +634,7 @@ public EventID getRootEventId() { .description("Root event") .status(Event.Status.PASSED) .type("Microservice") - .toProto(boxConfiguration.getBookName()); + .toProto(boxConfiguration.getBookName(), boxConfiguration.getBoxName()); try { getEventBatchRouter().sendAll(EventBatch.newBuilder().addEvents(rootEvent).build()); From 945ab74326f30ed5e47501838dde0634c2973147 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 9 Jan 2023 11:53:02 +0400 Subject: [PATCH 091/154] [TH2-4262] Use Suppliers.memoize to avoid re-subscribtion to a queue --- .../rabbitmq/AbstractRabbitSubscriber.java | 99 ++++++++++--------- 1 file changed, 55 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index fa7042d5b..38b4782d6 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -25,6 +25,7 @@ import com.exactpro.th2.common.schema.message.configuration.RouterFilter; import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; +import com.google.common.base.Suppliers; import com.google.protobuf.Message; import com.rabbitmq.client.Delivery; import io.prometheus.client.Counter; @@ -43,6 +44,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Supplier; import static com.exactpro.th2.common.metrics.CommonMetrics.DEFAULT_BUCKETS; import static com.exactpro.th2.common.metrics.CommonMetrics.QUEUE_LABEL; @@ -53,6 +55,9 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRabbitSubscriber.class); + @SuppressWarnings("rawtypes") + private static final Supplier EMPTY_INITIALIZER = Suppliers.memoize(() -> null); + private static final Counter MESSAGE_SIZE_SUBSCRIBE_BYTES = Counter.build() .name("th2_rabbitmq_message_size_subscribe_bytes") .labelNames(TH2_PIN_LABEL, TH2_TYPE_LABEL, QUEUE_LABEL) @@ -76,7 +81,7 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber> listeners = ConcurrentHashMap.newKeySet(); private final String queue; private final ConnectionManager connectionManager; - private final AtomicReference consumerMonitor = new AtomicReference<>(); + private final AtomicReference> consumerMonitor = new AtomicReference<>(emptySupplier()); private final AtomicReference filterFunc = new AtomicReference<>(); private final String th2Type; @@ -150,7 +155,7 @@ public void removeListener(ConfirmationListener messageListener) { public void close() throws IOException { publicLock.lock(); try { - SubscriberMonitor monitor = consumerMonitor.getAndSet(null); + SubscriberMonitor monitor = consumerMonitor.getAndSet(emptySupplier()).get(); if (monitor != null) { monitor.unsubscribe(); } @@ -225,46 +230,10 @@ protected void handle(DeliveryMetadata deliveryMetadata, Delivery delivery, T va private void subscribe() { try { - consumerMonitor.updateAndGet(monitor -> { - if (monitor == null) { - try { - monitor = connectionManager.basicConsume( - queue, - (deliveryMetadata, delivery, confirmProcessed) -> { - try (Timer ignored = MESSAGE_PROCESS_DURATION_SECONDS - .labels(th2Pin, th2Type, queue) - .startTimer()) { - MESSAGE_SIZE_SUBSCRIBE_BYTES - .labels(th2Pin, th2Type, queue) - .inc(delivery.getBody().length); - - T value; - try { - value = valueFromBytes(delivery.getBody()); - } catch (Exception e) { - LOGGER.error("Couldn't parse delivery. Reject message received", e); - confirmProcessed.reject(); - throw new IOException( - String.format( - "Can not extract value from bytes for envelope '%s', queue '%s', pin '%s'", - delivery.getEnvelope(), queue, th2Pin - ), - e - ); - } - handle(deliveryMetadata, delivery, value, confirmProcessed); - } - }, - this::canceled - ); - LOGGER.info("Start listening queue name='{}', th2 pin='{}'", queue, th2Pin); - } catch (IOException e) { - throw new IllegalStateException("Can not start subscribe to queue = " + queue, e); - } - } - - return monitor; - }); + consumerMonitor.updateAndGet(previous -> previous == EMPTY_INITIALIZER + ? Suppliers.memoize(this::basicConsume) + : previous) + .get(); // initialize subscribtion } catch (Exception e) { throw new IllegalStateException("Can not start listening", e); } @@ -273,7 +242,7 @@ private void subscribe() { private void resubscribe() { LOGGER.info("Try to resubscribe subscriber for queue name='{}'", queue); - SubscriberMonitor monitor = consumerMonitor.getAndSet(null); + SubscriberMonitor monitor = consumerMonitor.getAndSet(emptySupplier()).get(); if (monitor != null) { try { monitor.unsubscribe(); @@ -290,9 +259,51 @@ private void resubscribe() { } } + private SubscriberMonitor basicConsume() { + try { + LOGGER.info("Start listening queue name='{}', th2 pin='{}'", queue, th2Pin); + return connectionManager.basicConsume(queue, this::handle, this::canceled); + } catch (IOException e) { + throw new IllegalStateException("Can not subscribe to queue = " + queue, e); + } + } + + private void handle(DeliveryMetadata deliveryMetadata, + Delivery delivery, + Confirmation confirmProcessed) throws IOException { + try (Timer ignored = MESSAGE_PROCESS_DURATION_SECONDS + .labels(th2Pin, th2Type, queue) + .startTimer()) { + MESSAGE_SIZE_SUBSCRIBE_BYTES + .labels(th2Pin, th2Type, queue) + .inc(delivery.getBody().length); + + T value; + try { + value = valueFromBytes(delivery.getBody()); + } catch (Exception e) { + LOGGER.error("Couldn't parse delivery. Reject message received", e); + confirmProcessed.reject(); + throw new IOException( + String.format( + "Can not extract value from bytes for envelope '%s', queue '%s', pin '%s'", + delivery.getEnvelope(), queue, th2Pin + ), + e + ); + } + handle(deliveryMetadata, delivery, value, confirmProcessed); + } + } + private void canceled(String consumerTag) { LOGGER.warn("Consuming cancelled for: '{}'", consumerTag); healthMetrics.getReadinessMonitor().disable(); resubscribe(); } + + @SuppressWarnings("unchecked") + private static Supplier emptySupplier() { + return (Supplier) EMPTY_INITIALIZER; + } } From 87123bfc618195f5f1a702cefa8f939af7fdcb15 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Fri, 13 Jan 2023 18:02:30 +0400 Subject: [PATCH 092/154] [TH2-4262] Migrated to grpc-common 4.1.0 * Extended MessageRouter.storeEvent method --- README.md | 8 ++++++-- build.gradle | 2 +- gradle.properties | 2 +- .../th2/common/schema/message/MessageRouterUtils.kt | 7 ++++--- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 34227216b..285dffd01 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (4.0.0+) +# th2 common library (Java) (5.1.0+) ## Usage @@ -361,7 +361,11 @@ dependencies { ## Release notes -### 4.0.0 +### 5.1.0 + ++ Migrated to grpc-common 4.1.0 + +### 5.0.0 + Migration to books/pages cradle 4.0.0 + Migration to bom 4.0.2 diff --git a/build.gradle b/build.gradle index 9a23595b7..1df112949 100644 --- a/build.gradle +++ b/build.gradle @@ -173,7 +173,7 @@ dependencies { api ("com.exactpro.th2:cradle-core:${cradleVersion}") { exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability } - api 'com.exactpro.th2:grpc-common:4.0.0-TH2-4262-reduce-load-book-and-page-3574839736-SNAPSHOT' + api 'com.exactpro.th2:grpc-common:4.1.0-th2-2150-books-pages-3871780258-SNAPSHOT' jmh 'org.openjdk.jmh:jmh-core:0.9' jmh 'org.openjdk.jmh:jmh-generator-annprocess:0.9' diff --git a/gradle.properties b/gradle.properties index 7236c615c..4386fa6a0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=5.0.0 +release_version=5.1.0 description = 'th2 common library (Java)' diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt index 7a4f7f49d..677218cf4 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/MessageRouterUtils.kt @@ -39,11 +39,12 @@ fun MessageRouter.storeEvent( parentId: EventID ) = storeEvent(event, event.toProto(parentId)) - +@JvmOverloads fun MessageRouter.storeEvent( event: Event, - bookName: String -) = storeEvent(event, event.toProto(bookName)) + bookName: String, + scope: String? = null +) = storeEvent(event, event.toProto(bookName, scope)) private fun MessageRouter.storeEvent( event: Event, From 576387b797153b495e35342797156ffaa917a440 Mon Sep 17 00:00:00 2001 From: Fiodar Rekish <38082705+Xanclry@users.noreply.github.com> Date: Mon, 16 Jan 2023 13:04:05 +0400 Subject: [PATCH 093/154] [TH2-4566] reusable workflow with vulnerabilities scanning (#245) * reusable workflow, dependency check and vulnerabilities fix Co-authored-by: fiodar.rekish Co-authored-by: Denis Plotnikov --- .../workflows/dev-java-publish-sonatype.yml | 49 ++++----------- .github/workflows/java-publish-sonatype.yml | 24 +++----- README.md | 7 ++- build.gradle | 9 ++- gradle.properties | 2 +- .../schema/factory/AbstractCommonFactory.java | 3 +- .../common/schema/factory/CommonFactory.java | 6 +- .../common/schema/util/Log4jConfigUtils.kt | 59 +++++++------------ 8 files changed, 57 insertions(+), 102 deletions(-) diff --git a/.github/workflows/dev-java-publish-sonatype.yml b/.github/workflows/dev-java-publish-sonatype.yml index 4b6e75dc6..ec39beb2f 100644 --- a/.github/workflows/dev-java-publish-sonatype.yml +++ b/.github/workflows/dev-java-publish-sonatype.yml @@ -12,42 +12,13 @@ on: - README.md jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 -# Prepare custom build version - - name: Get branch name - id: branch - run: echo ::set-output name=branch_name::${GITHUB_REF#refs/*/} - - name: Get release_version - id: ver - uses: christian-draeger/read-properties@1.0.1 - with: - path: gradle.properties - property: release_version - - name: Build custom release version - id: release_ver - run: echo ::set-output name=value::"${{ steps.ver.outputs.value }}-${{ steps.branch.outputs.branch_name }}-${{ github.run_id }}-SNAPSHOT" - - name: Write custom release version to file - uses: christian-draeger/write-properties@1.0.1 - with: - path: gradle.properties - property: release_version - value: ${{ steps.release_ver.outputs.value }} - - name: Show custom release version - run: echo ${{ steps.release_ver.outputs.value }} -# Build and publish package - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - - name: Build with Gradle - run: ./gradlew --info clean build publish - env: - ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} - ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} - ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} - ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} - + build-job: + uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main + with: + build-target: 'Sonatype' + runsOn: ubuntu-latest + secrets: + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} diff --git a/.github/workflows/java-publish-sonatype.yml b/.github/workflows/java-publish-sonatype.yml index b8c366887..aab717525 100644 --- a/.github/workflows/java-publish-sonatype.yml +++ b/.github/workflows/java-publish-sonatype.yml @@ -10,18 +10,12 @@ on: jobs: build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - - name: Build with Gradle - run: ./gradlew --info clean build publish closeAndReleaseSonatypeStagingRepository - env: - ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} - ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} - ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} - ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Sonatype' + runsOn: ubuntu-latest + secrets: + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} diff --git a/README.md b/README.md index 711cbaff2..966a88097 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (3.42.0) +# th2 common library (Java) (3.43.0) ## Usage @@ -357,6 +357,11 @@ dependencies { ## Release notes +### 3.43.0 ++ There is no support for log4j version 1. ++ Work was done to eliminate vulnerabilities in _common_ and _bom_ dependencies. + + **th2-bom must be updated to _4.0.3_ or higher.** + ### 3.42.0 + Added the `enableSizeMeasuring`, `maxMessageSize`, `keepAliveInterval` options into gRPC router configuration. Default values are false, 4194304, 60 diff --git a/build.gradle b/build.gradle index 579c0ec28..f8147aa6f 100644 --- a/build.gradle +++ b/build.gradle @@ -55,6 +55,12 @@ repositories { } } +configurations { + compileClasspath { + resolutionStrategy.activateDependencyLocking() + } +} + java { withJavadocJar() withSourcesJar() @@ -169,7 +175,7 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:4.0.2") + api platform("com.exactpro.th2:bom:4.0.3") api ("com.exactpro.th2:cradle-core:${cradleVersion}") { exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability } @@ -215,7 +221,6 @@ dependencies { api 'io.github.microutils:kotlin-logging:2.1.21' implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' - implementation 'org.apache.logging.log4j:log4j-1.2-api' implementation 'org.apache.logging.log4j:log4j-core' implementation 'io.prometheus:simpleclient' diff --git a/gradle.properties b/gradle.properties index d9abe0345..bb25870a8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.42.0 +release_version=3.43.0 description = 'th2 common library (Java)' diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index ba074c6be..60aa52e97 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -117,7 +117,6 @@ public abstract class AbstractCommonFactory implements AutoCloseable { protected static final String LOG4J_PROPERTIES_DEFAULT_PATH_OLD = "/home/etc"; protected static final String LOG4J_PROPERTIES_DEFAULT_PATH = "/var/th2/config"; protected static final String LOG4J2_PROPERTIES_NAME = "log4j2.properties"; - protected static final String LOG4J_PROPERTIES_NAME = "log4j.properties"; protected static final ObjectMapper MAPPER = new ObjectMapper(); @@ -769,7 +768,7 @@ protected static void configureLogger(String... paths) { listPath.add(LOG4J_PROPERTIES_DEFAULT_PATH_OLD); listPath.addAll(Arrays.asList(requireNonNull(paths, "Paths can't be null"))); Log4jConfigUtils log4jConfigUtils = new Log4jConfigUtils(); - log4jConfigUtils.configure(listPath, LOG4J_PROPERTIES_NAME, LOG4J2_PROPERTIES_NAME); + log4jConfigUtils.configure(listPath, LOG4J2_PROPERTIES_NAME); loggingManifests(); } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index 399eb21f8..9d7645b9c 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -454,8 +454,8 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam Map boxData = boxConfigMap.getData(); Map rabbitMqData = rabbitMqConfigMap.getData(); Map cradleConfigData = cradleConfigMap.getData(); - @Nullable String loggingData = boxData.getOrDefault(LOG4J_PROPERTIES_NAME, - loggingConfigMap == null ? null : loggingConfigMap.getData().get(LOG4J_PROPERTIES_NAME) + @Nullable String loggingData = boxData.getOrDefault(LOG4J2_PROPERTIES_NAME, + loggingConfigMap == null ? null : loggingConfigMap.getData().get(LOG4J2_PROPERTIES_NAME) ); File generatedConfigsDirFile = configPath.toFile(); @@ -471,7 +471,7 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam box.setBoxName(boxName); if (loggingData != null) { - writeFile(configPath.resolve(LOG4J_PROPERTIES_NAME), loggingData); + writeFile(configPath.resolve(LOG4J2_PROPERTIES_NAME), loggingData); configureLogger(configPath.toString()); } diff --git a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt index d6e081c10..5763a0fdf 100644 --- a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt +++ b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt @@ -15,13 +15,11 @@ package com.exactpro.th2.common.schema.util -import org.apache.log4j.PropertyConfigurator -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.core.LoggerContext -import org.slf4j.LoggerFactory import java.net.MalformedURLException import java.nio.file.Files import java.nio.file.Path +import org.apache.logging.log4j.core.LoggerContext +import org.slf4j.LoggerFactory class Log4jConfigUtils { @@ -29,47 +27,30 @@ class Log4jConfigUtils { fun configure( pathList: List, - firstVersionFileName: String, - secondVersionFileName: String, + fileName: String, ) { - val configPathList: Pair? = pathList.asSequence() - .flatMap { - listOf( - Pair(Path.of(it, firstVersionFileName), 1), - Pair(Path.of(it, secondVersionFileName), 2) - ) - } - .filter { Files.exists(it.first) } - .sortedByDescending { it.second } + pathList.asSequence() + .map { Path.of(it, fileName) } + .filter(Files::exists) .firstOrNull() - - when (configPathList?.second) { - 1 -> configureFirstLog4j(configPathList.first) - 2 -> configureSecondLog4j(configPathList.first) - null -> { + ?.let { path -> + try { + LOGGER.info("Trying to apply logger config from {}. Expecting log4j syntax", path) + val loggerContext = LoggerContext.getContext(false) + loggerContext.configLocation = path.toUri() + loggerContext.reconfigure() + LOGGER.info("Logger configuration from {} file is applied", path) + } catch (e: MalformedURLException) { + LOGGER.error(e.message, e) + } + } + ?: run { LOGGER.info( - "Neither of {} paths contains config files {}, {}. Use default configuration", + "Neither of {} paths contains config file {}. Use default configuration", pathList, - firstVersionFileName, - secondVersionFileName + fileName ) } - } - } - - private fun configureFirstLog4j(path: Path) { - try { - PropertyConfigurator.configure(path.toUri().toURL()) - LOGGER.info("Logger configuration from {} file is applied", path) - } catch (e: MalformedURLException) { - e.printStackTrace() - } - } - - private fun configureSecondLog4j(path: Path) { - val context = LogManager.getContext(false) as LoggerContext - context.configLocation = path.toUri() - LOGGER.info("Logger configuration from {} file is applied", path) } } \ No newline at end of file From 4a7d26a05f5b319cfcc5536a34d14cd9d4039bc7 Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Mon, 16 Jan 2023 19:43:50 +0400 Subject: [PATCH 094/154] medium severity vulnerabilities fix (#246) * bump bom version --- README.md | 2 +- build.gradle | 2 +- gradle.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 966a88097..8f36ef998 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (3.43.0) +# th2 common library (Java) (3.44.0) ## Usage diff --git a/build.gradle b/build.gradle index f8147aa6f..0f0061819 100644 --- a/build.gradle +++ b/build.gradle @@ -175,7 +175,7 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:4.0.3") + api platform("com.exactpro.th2:bom:4.1.0") api ("com.exactpro.th2:cradle-core:${cradleVersion}") { exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability } diff --git a/gradle.properties b/gradle.properties index bb25870a8..d5d4ac5d1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.43.0 +release_version=3.44.0 description = 'th2 common library (Java)' From 63fc7a79d36980a47ed9ae110a6856dd5aa84f03 Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Fri, 20 Jan 2023 17:19:30 +0400 Subject: [PATCH 095/154] [TH2-2743] MessageUtil > Add bookName extension property for RawMessage --- .../com/exactpro/th2/common/message/MessageUtils.kt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt index b22d3266d..d0fbf1da4 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt @@ -307,6 +307,14 @@ var Message.Builder.bookName metadataBuilder.idBuilder.bookName = value } +val RawMessage.bookName + get(): String = metadata.id.bookName +var RawMessage.Builder.bookName + get(): String = metadata.id.bookName + set(value) { + metadataBuilder.idBuilder.bookName = value + } + val Message.messageType get(): String = metadata.messageType var Message.Builder.messageType From 8adee332b822bbdf59537853e79287e9411a4edf Mon Sep 17 00:00:00 2001 From: isengrims <104489572+isengrims@users.noreply.github.com> Date: Fri, 3 Feb 2023 18:22:11 +0400 Subject: [PATCH 096/154] Merge branch `master` into `dev-version-5` (#249) --- .../workflows/dev-java-publish-sonatype.yml | 49 ++++--------------- .github/workflows/java-publish-sonatype.yml | 24 ++++----- README.md | 5 ++ build.gradle | 8 ++- .../schema/factory/AbstractCommonFactory.java | 1 + .../common/schema/util/Log4jConfigUtils.kt | 10 ++-- 6 files changed, 37 insertions(+), 60 deletions(-) diff --git a/.github/workflows/dev-java-publish-sonatype.yml b/.github/workflows/dev-java-publish-sonatype.yml index 4b6e75dc6..ec39beb2f 100644 --- a/.github/workflows/dev-java-publish-sonatype.yml +++ b/.github/workflows/dev-java-publish-sonatype.yml @@ -12,42 +12,13 @@ on: - README.md jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 -# Prepare custom build version - - name: Get branch name - id: branch - run: echo ::set-output name=branch_name::${GITHUB_REF#refs/*/} - - name: Get release_version - id: ver - uses: christian-draeger/read-properties@1.0.1 - with: - path: gradle.properties - property: release_version - - name: Build custom release version - id: release_ver - run: echo ::set-output name=value::"${{ steps.ver.outputs.value }}-${{ steps.branch.outputs.branch_name }}-${{ github.run_id }}-SNAPSHOT" - - name: Write custom release version to file - uses: christian-draeger/write-properties@1.0.1 - with: - path: gradle.properties - property: release_version - value: ${{ steps.release_ver.outputs.value }} - - name: Show custom release version - run: echo ${{ steps.release_ver.outputs.value }} -# Build and publish package - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - - name: Build with Gradle - run: ./gradlew --info clean build publish - env: - ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} - ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} - ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} - ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} - + build-job: + uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main + with: + build-target: 'Sonatype' + runsOn: ubuntu-latest + secrets: + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} diff --git a/.github/workflows/java-publish-sonatype.yml b/.github/workflows/java-publish-sonatype.yml index b8c366887..aab717525 100644 --- a/.github/workflows/java-publish-sonatype.yml +++ b/.github/workflows/java-publish-sonatype.yml @@ -10,18 +10,12 @@ on: jobs: build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - - name: Build with Gradle - run: ./gradlew --info clean build publish closeAndReleaseSonatypeStagingRepository - env: - ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} - ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} - ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} - ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Sonatype' + runsOn: ubuntu-latest + secrets: + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} diff --git a/README.md b/README.md index 285dffd01..1f3a3c90e 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,11 @@ dependencies { + `com.exactpro.th2.common.event.Event.toProto...()` by `parentEventId`/`bookName`/`(bookName + scope)` + Added `isRedelivered` flag to message +### 3.43.0 ++ There is no support for log4j version 1. ++ Work was done to eliminate vulnerabilities in _common_ and _bom_ dependencies. + + **th2-bom must be updated to _4.0.3_ or higher.** + ### 3.42.0 + Added the `enableSizeMeasuring`, `maxMessageSize`, `keepAliveInterval` options into gRPC router configuration. Default values are false, 4194304, 60 diff --git a/build.gradle b/build.gradle index 1df112949..d5e8ea435 100644 --- a/build.gradle +++ b/build.gradle @@ -55,6 +55,12 @@ repositories { } } +configurations { + compileClasspath { + resolutionStrategy.activateDependencyLocking() + } +} + java { withJavadocJar() withSourcesJar() @@ -169,7 +175,7 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform('com.exactpro.th2:bom:4.0.2') + api platform("com.exactpro.th2:bom:4.1.0") api ("com.exactpro.th2:cradle-core:${cradleVersion}") { exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 13e9f13ed..643d5ceeb 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -113,6 +113,7 @@ public abstract class AbstractCommonFactory implements AutoCloseable { protected static final String LOG4J_PROPERTIES_DEFAULT_PATH_OLD = "/home/etc"; protected static final String LOG4J_PROPERTIES_DEFAULT_PATH = "/var/th2/config"; protected static final String LOG4J2_PROPERTIES_NAME = "log4j2.properties"; + protected static final ObjectMapper MAPPER = new ObjectMapper(); static { diff --git a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt index 113b6e870..b8ca23b45 100644 --- a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt +++ b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt @@ -15,20 +15,20 @@ package com.exactpro.th2.common.schema.util -import org.apache.logging.log4j.core.LoggerContext -import org.slf4j.LoggerFactory import java.net.MalformedURLException import java.nio.file.Files import java.nio.file.Path +import org.apache.logging.log4j.core.LoggerContext +import org.slf4j.LoggerFactory class Log4jConfigUtils { fun configure( pathList: List, - versionFileName: String + fileName: String, ) { pathList.asSequence() - .map { Path.of(it, versionFileName) } + .map { Path.of(it, fileName) } .filter(Files::exists) .firstOrNull() ?.let { path -> @@ -46,7 +46,7 @@ class Log4jConfigUtils { LOGGER.info( "Neither of {} paths contains config file {}. Use default configuration", pathList, - versionFileName + fileName ) } } From 176471c8c5d7a2d6c230b746eabff0221792c82a Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Fri, 17 Feb 2023 11:16:54 +0400 Subject: [PATCH 097/154] [TH2-4778] Common cradle non-confidential config new parameters and bugfix (#252) * [TH2-4778] added statisticsPersistenceIntervalMillis in cradle config * [TH2-4778] fix * [TH2-4778] changed cradle non-confidential generation while using kubernetes * [TH2-4778] provided compression configuration * [TH2-4778] fixed build * [TH2-4778] changed test * [TH2-4778] merge fix --- .../schema/factory/AbstractCommonFactory.java | 9 +++++++++ .../th2/common/schema/factory/CommonFactory.java | 14 +++++++++----- .../common/schema/cradle/CradleConfiguration.kt | 15 ++++++++++++--- .../th2/common/schema/TestJsonConfiguration.kt | 5 ++++- .../cradle_non_confidential.json | 5 ++++- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 643d5ceeb..f25b06c7f 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -532,6 +532,15 @@ public CradleManager getCradleManager() { if (nonConfidentialConfiguration.getCradleMaxEventBatchSize() > 0) { cassandraStorageSettings.setMaxTestEventBatchSize((int) nonConfidentialConfiguration.getCradleMaxEventBatchSize()); } + if (nonConfidentialConfiguration.getStatisticsPersistenceIntervalMillis() > 0) { + cassandraStorageSettings.setCounterPersistenceInterval((int) nonConfidentialConfiguration.getStatisticsPersistenceIntervalMillis()); + } + if (nonConfidentialConfiguration.getMaxUncompressedMessageBatchSize() > 0) { + cassandraStorageSettings.setMaxUncompressedMessageBatchSize((int) nonConfidentialConfiguration.getMaxUncompressedMessageBatchSize()); + } + if (nonConfidentialConfiguration.getMaxUncompressedEventBatchSize() > 0) { + cassandraStorageSettings.setMaxUncompressedTestEventSize((int) nonConfidentialConfiguration.getMaxUncompressedEventBatchSize()); + } manager = new CassandraCradleManager( cassandraConnectionSettings, diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index c4f5811e5..f3e3959ad 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -115,6 +115,7 @@ public class CommonFactory extends AbstractCommonFactory { private static final String GENERATED_CONFIG_DIR_NAME = "generated_configs"; private static final String RABBIT_MQ_EXTERNAL_APP_CONFIG_MAP = "rabbit-mq-external-app-config"; private static final String CRADLE_EXTERNAL_MAP = "cradle-external"; + private static final String CRADLE_MANAGER_CONFIG_MAP = "cradle-manager"; private static final String LOGGING_CONFIG_MAP = "logging-config"; private final Path custom; @@ -444,17 +445,20 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam throw new IllegalArgumentException("Failed to find config maps by boxName " + boxName); } Resource rabbitMqConfigMapResource = configMaps.inNamespace(namespace).withName(RABBIT_MQ_EXTERNAL_APP_CONFIG_MAP); - Resource cradleConfigMapResource = configMaps.inNamespace(namespace).withName(CRADLE_EXTERNAL_MAP); + Resource cradleConfidentialConfigMapResource = configMaps.inNamespace(namespace).withName(CRADLE_EXTERNAL_MAP); + Resource cradleNonConfidentialConfigMapResource = configMaps.inNamespace(namespace).withName(CRADLE_MANAGER_CONFIG_MAP); Resource loggingConfigMapResource = configMaps.inNamespace(namespace).withName(LOGGING_CONFIG_MAP); ConfigMap boxConfigMap = boxConfigMapResource.require(); ConfigMap rabbitMqConfigMap = rabbitMqConfigMapResource.require(); - ConfigMap cradleConfigMap = cradleConfigMapResource.require(); + ConfigMap cradleConfidentialConfigmap = cradleConfidentialConfigMapResource.require(); + ConfigMap cradleNonConfidentialConfigmap = cradleNonConfidentialConfigMapResource.require(); @Nullable ConfigMap loggingConfigMap = loggingConfigMapResource.get(); Map boxData = boxConfigMap.getData(); Map rabbitMqData = rabbitMqConfigMap.getData(); - Map cradleConfigData = cradleConfigMap.getData(); + Map cradleConfidential = cradleConfidentialConfigmap.getData(); + Map cradleNonConfidential = cradleNonConfidentialConfigmap.getData(); @Nullable String loggingData = boxData.getOrDefault(LOG4J2_PROPERTIES_NAME, loggingConfigMap == null ? null : loggingConfigMap.getData().get(LOG4J2_PROPERTIES_NAME) ); @@ -481,8 +485,8 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam settings.setConnectionManagerSettings(writeFile(configPath, CONNECTION_MANAGER_CONF_FILE_NAME, boxData)); settings.setGrpc(writeFile(configPath, GRPC_FILE_NAME, boxData)); settings.setRouterGRPC(writeFile(configPath, ROUTER_GRPC_FILE_NAME, boxData)); - settings.setCradleConfidential(writeFile(configPath, CRADLE_CONFIDENTIAL_FILE_NAME, cradleConfigData)); - settings.setCradleNonConfidential(writeFile(configPath, CRADLE_NON_CONFIDENTIAL_FILE_NAME, boxData)); + settings.setCradleConfidential(writeFile(configPath, CRADLE_CONFIDENTIAL_FILE_NAME, cradleConfidential)); + settings.setCradleNonConfidential(writeFile(configPath, CRADLE_NON_CONFIDENTIAL_FILE_NAME, cradleNonConfidential)); settings.setPrometheus(writeFile(configPath, PROMETHEUS_FILE_NAME, boxData)); settings.setCustom(writeFile(configPath, CUSTOM_FILE_NAME, boxData)); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt index 572e078f6..f852b8128 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt @@ -32,7 +32,10 @@ data class CradleConfiguration( var pageSize: Int, var cradleMaxEventBatchSize: Long, var cradleMaxMessageBatchSize: Long, - var prepareStorage: Boolean + var prepareStorage: Boolean, + var statisticsPersistenceIntervalMillis: Long, + var maxUncompressedMessageBatchSize: Long, + var maxUncompressedEventBatchSize: Long ) : Configuration() { constructor( cradleConfidentialConfiguration: CradleConfidentialConfiguration, @@ -48,7 +51,10 @@ data class CradleConfiguration( cradleNonConfidentialConfiguration.pageSize, cradleNonConfidentialConfiguration.cradleMaxEventBatchSize, cradleNonConfidentialConfiguration.cradleMaxMessageBatchSize, - cradleNonConfidentialConfiguration.prepareStorage + cradleNonConfidentialConfiguration.prepareStorage, + cradleNonConfidentialConfiguration.statisticsPersistenceIntervalMillis, + cradleNonConfidentialConfiguration.maxUncompressedMessageBatchSize, + cradleNonConfidentialConfiguration.maxUncompressedEventBatchSize ) } @@ -66,5 +72,8 @@ data class CradleNonConfidentialConfiguration( var pageSize: Int = 5000, var cradleMaxEventBatchSize: Long = CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE.toLong(), var cradleMaxMessageBatchSize: Long = CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE.toLong(), - var prepareStorage: Boolean = false + var prepareStorage: Boolean = false, + var statisticsPersistenceIntervalMillis: Long = CassandraStorageSettings.DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS.toLong(), + var maxUncompressedMessageBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_MESSAGE_BATCH_SIZE.toLong(), + var maxUncompressedEventBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_TEST_EVENT_SIZE.toLong() ) : Configuration() \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index c0188af94..3f014fb00 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -220,7 +220,10 @@ class TestJsonConfiguration { 111, 123, 321, - false + false, + 5000, + 1280001, + 1280002 ) private val PROMETHEUS_CONF_JSON = loadConfJson("prometheus") diff --git a/src/test/resources/test_json_configurations/cradle_non_confidential.json b/src/test/resources/test_json_configurations/cradle_non_confidential.json index e9967c45b..874aa01f4 100644 --- a/src/test/resources/test_json_configurations/cradle_non_confidential.json +++ b/src/test/resources/test_json_configurations/cradle_non_confidential.json @@ -3,5 +3,8 @@ "pageSize": 111, "cradleMaxEventBatchSize": 123, "cradleMaxMessageBatchSize": 321, - "prepareStorage": false + "prepareStorage": false, + "statisticsPersistenceIntervalMillis": 5000, + "maxUncompressedMessageBatchSize": 1280001, + "maxUncompressedEventBatchSize": 1280002 } \ No newline at end of file From 5c859066587cb5ffc611f8ae8363132a6a0fc0b7 Mon Sep 17 00:00:00 2001 From: eugene-zheltov <75071188+eugene-zheltov@users.noreply.github.com> Date: Wed, 22 Feb 2023 12:02:25 +0400 Subject: [PATCH 098/154] Add dev-release github action (#256) * Add dev-release github action --------- Co-authored-by: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> --- .../workflows/dev-java-publish-sonatype.yml | 1 + .../dev-release-java-publish-sonatype.yml | 22 +++++++++++++++++++ README.md | 8 ++++++- build.gradle | 4 ++-- gradle.properties | 2 +- 5 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dev-release-java-publish-sonatype.yml diff --git a/.github/workflows/dev-java-publish-sonatype.yml b/.github/workflows/dev-java-publish-sonatype.yml index ec39beb2f..488934c05 100644 --- a/.github/workflows/dev-java-publish-sonatype.yml +++ b/.github/workflows/dev-java-publish-sonatype.yml @@ -5,6 +5,7 @@ on: branches-ignore: - master - version-* + - dev-version-* - dependabot* # paths: # - gradle.properties diff --git a/.github/workflows/dev-release-java-publish-sonatype.yml b/.github/workflows/dev-release-java-publish-sonatype.yml new file mode 100644 index 000000000..5d2433d96 --- /dev/null +++ b/.github/workflows/dev-release-java-publish-sonatype.yml @@ -0,0 +1,22 @@ +name: Build and publish dev-release Java distributions to sonatype. + +on: + workflow_dispatch: + push: + branches: + - dev-version-* + paths: + - gradle.properties + +jobs: + build: + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Sonatype' + runsOn: ubuntu-latest + devRelease: true + secrets: + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} \ No newline at end of file diff --git a/README.md b/README.md index 1f3a3c90e..668013aff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (5.1.0+) +# th2 common library (Java) (5.1.1) ## Usage @@ -361,6 +361,12 @@ dependencies { ## Release notes +### 5.1.1 + ++ Added script for publishing dev-release for maven artefacts ++ Migrated to bom:4.2.0 ++ Migrated to grpc-common:4.1.1-dev + ### 5.1.0 + Migrated to grpc-common 4.1.0 diff --git a/build.gradle b/build.gradle index d5e8ea435..9ce449113 100644 --- a/build.gradle +++ b/build.gradle @@ -175,11 +175,11 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:4.1.0") + api platform("com.exactpro.th2:bom:4.2.0") api ("com.exactpro.th2:cradle-core:${cradleVersion}") { exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability } - api 'com.exactpro.th2:grpc-common:4.1.0-th2-2150-books-pages-3871780258-SNAPSHOT' + api 'com.exactpro.th2:grpc-common:4.1.1-dev' jmh 'org.openjdk.jmh:jmh-core:0.9' jmh 'org.openjdk.jmh:jmh-generator-annprocess:0.9' diff --git a/gradle.properties b/gradle.properties index 4386fa6a0..ed1ef4a40 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=5.1.0 +release_version=5.1.1 description = 'th2 common library (Java)' From 43afb8e3ba9ef5c58eb4d1813b0401cb0d0f6a85 Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Wed, 22 Feb 2023 21:44:30 +0400 Subject: [PATCH 099/154] Vulnerabilities (#257) * Removed jackson-dataformat-yaml * Corrected build.gradle configuration --- README.md | 6 +++++- build.gradle | 16 ++++++++-------- gradle.properties | 2 +- .../exactpro/th2/common/ConfigurationUtils.java | 8 ++++---- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8f36ef998..e26c2f7c5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (3.44.0) +# th2 common library (Java) (3.44.1) ## Usage @@ -357,6 +357,10 @@ dependencies { ## Release notes +### 3.44.1 ++ Remove unused dependency ++ Updated bom:4.2.0 + ### 3.43.0 + There is no support for log4j version 1. + Work was done to eliminate vulnerabilities in _common_ and _bom_ dependencies. diff --git a/build.gradle b/build.gradle index 0f0061819..787079c11 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,7 @@ plugins { id 'signing' id 'com.google.protobuf' version '0.8.8' apply false id 'org.jetbrains.kotlin.jvm' version "${kotlin_version}" - id "org.owasp.dependencycheck" version "7.2.0" + id "org.owasp.dependencycheck" version "8.1.0" id "me.champeau.jmh" version "0.6.8" } @@ -175,11 +175,12 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:4.1.0") + api platform("com.exactpro.th2:bom:4.2.0") + api 'com.exactpro.th2:grpc-common:3.11.1' api ("com.exactpro.th2:cradle-core:${cradleVersion}") { exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability } - api 'com.exactpro.th2:grpc-common:3.11.1' + api 'io.github.microutils:kotlin-logging:2.1.21' jmh 'org.openjdk.jmh:jmh-core:0.9' jmh 'org.openjdk.jmh:jmh-generator-annprocess:0.9' @@ -213,13 +214,10 @@ dependencies { because('providee ability to use jackson annotations in components') } implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" implementation "com.fasterxml.jackson.module:jackson-module-kotlin" implementation 'com.fasterxml.uuid:java-uuid-generator:4.0.1' - api 'io.github.microutils:kotlin-logging:2.1.21' - implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' implementation 'org.apache.logging.log4j:log4j-core' @@ -231,7 +229,9 @@ dependencies { implementation ('com.squareup.okhttp3:okhttp:4.10.0') { because ('fix vulnerability in transitive dependency ') } - implementation 'io.fabric8:kubernetes-client:6.1.1' + implementation ('io.fabric8:kubernetes-client:6.1.1') { + exclude group: 'com.fasterxml.jackson.dataformat', module: 'jackson-dataformat-yaml' + } testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" testImplementation 'org.mockito.kotlin:mockito-kotlin:4.0.0' @@ -269,7 +269,7 @@ clean { } dependencyCheck { - formats=['JSON', 'HTML'] + formats=['SARIF', 'JSON', 'HTML'] failBuildOnCVSS=5 analyzers { diff --git a/gradle.properties b/gradle.properties index d5d4ac5d1..612ec16a8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # limitations under the License. # -release_version=3.44.0 +release_version=3.44.1 description = 'th2 common library (Java)' diff --git a/src/main/java/com/exactpro/th2/common/ConfigurationUtils.java b/src/main/java/com/exactpro/th2/common/ConfigurationUtils.java index 1259c843a..42babd6b0 100644 --- a/src/main/java/com/exactpro/th2/common/ConfigurationUtils.java +++ b/src/main/java/com/exactpro/th2/common/ConfigurationUtils.java @@ -24,6 +24,7 @@ import java.nio.file.Paths; import java.util.function.Supplier; +import com.fasterxml.jackson.core.JsonFactory; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -31,12 +32,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; public class ConfigurationUtils { private static final Logger LOGGER = LoggerFactory.getLogger(ConfigurationUtils.class); - private static final ObjectMapper YAML_READER = new ObjectMapper(new YAMLFactory()); + private static final ObjectMapper JSON_READER = new ObjectMapper(new JsonFactory()); @Nullable public static String getEnv(String key, @Nullable String defaultValue) { @@ -47,7 +47,7 @@ public static String getEnv(String key, @Nullable String defaultValue) { public static T getJsonEnv(String key, TypeReference cls) { String env = getEnv(key, null); try { - return env == null ? null : YAML_READER.readValue(env, cls); + return env == null ? null : JSON_READER.readValue(env, cls); } catch (IOException e) { LOGGER.error("Can not parse json environment variable with key: " + key, e); return null; @@ -55,7 +55,7 @@ public static T getJsonEnv(String key, TypeReference cls) { } public static T load(Class _class, InputStream inputStream) throws IOException { - return YAML_READER.readValue(inputStream, _class); + return JSON_READER.readValue(inputStream, _class); } /** From 9366b966d9fbb3b0b22f544fe30e0ae502e55a43 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 28 Feb 2023 11:19:27 +0400 Subject: [PATCH 100/154] Updated owasp plugin to 8.1.1 --- build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 184e7eef2..e47d9d00c 100644 --- a/build.gradle +++ b/build.gradle @@ -21,16 +21,13 @@ plugins { id 'signing' id 'com.google.protobuf' version '0.8.8' apply false id 'org.jetbrains.kotlin.jvm' version "${kotlin_version}" - id "org.owasp.dependencycheck" version "8.1.0" + id "org.owasp.dependencycheck" version "8.1.1" id "me.champeau.jmh" version "0.6.8" } group = 'com.exactpro.th2' version = release_version -sourceCompatibility = 11 -targetCompatibility = 11 - ext { cradleVersion = '5.0.0-dev-version-5-2567311063-SNAPSHOT' junitVersion = '5.8.2' @@ -62,6 +59,9 @@ configurations { } java { + sourceCompatibility = 11 + targetCompatibility = 11 + withJavadocJar() withSourcesJar() } From d53aaa06f39b932bc77c389991a832f96e77034e Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Tue, 28 Feb 2023 13:08:14 +0400 Subject: [PATCH 101/154] [TH2-4778] changed logic to set persistence interval (#259) --- .../th2/common/schema/factory/AbstractCommonFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index f25b06c7f..7e8235257 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -532,7 +532,7 @@ public CradleManager getCradleManager() { if (nonConfidentialConfiguration.getCradleMaxEventBatchSize() > 0) { cassandraStorageSettings.setMaxTestEventBatchSize((int) nonConfidentialConfiguration.getCradleMaxEventBatchSize()); } - if (nonConfidentialConfiguration.getStatisticsPersistenceIntervalMillis() > 0) { + if (nonConfidentialConfiguration.getStatisticsPersistenceIntervalMillis() >= 0) { cassandraStorageSettings.setCounterPersistenceInterval((int) nonConfidentialConfiguration.getStatisticsPersistenceIntervalMillis()); } if (nonConfidentialConfiguration.getMaxUncompressedMessageBatchSize() > 0) { From aa0584b4bda1323a22d812d34a2ce5b5469710fa Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 6 Mar 2023 13:41:18 +0400 Subject: [PATCH 102/154] Added ComposingServiceThreads option --- build.gradle | 6 ++++-- .../th2/common/schema/factory/AbstractCommonFactory.java | 3 +++ .../th2/common/schema/cradle/CradleConfiguration.kt | 1 + .../com/exactpro/th2/common/schema/TestJsonConfiguration.kt | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index e47d9d00c..cf8a519ab 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.0.0-dev-version-5-2567311063-SNAPSHOT' + cradleVersion = '5.0.0-separate-executor-4341698850-6476db6-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } @@ -187,9 +187,10 @@ dependencies { implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.2.2' - implementation ("com.exactpro.th2:cradle-cassandra:${cradleVersion}") { + api ("com.exactpro.th2:cradle-cassandra:${cradleVersion}") { exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability } + implementation "com.datastax.oss:java-driver-core" //FIXME: Add these dependencies as api to grpc-... artifacts implementation "io.grpc:grpc-protobuf" @@ -202,6 +203,7 @@ dependencies { implementation "org.jetbrains:annotations" implementation "org.apache.commons:commons-lang3" + implementation "commons-io:commons-io" implementation "org.apache.commons:commons-collections4" implementation "org.apache.commons:commons-text" implementation "commons-cli:commons-cli" diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 7e8235257..c43675e77 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -541,6 +541,9 @@ public CradleManager getCradleManager() { if (nonConfidentialConfiguration.getMaxUncompressedEventBatchSize() > 0) { cassandraStorageSettings.setMaxUncompressedTestEventSize((int) nonConfidentialConfiguration.getMaxUncompressedEventBatchSize()); } + if (nonConfidentialConfiguration.getComposingServiceThreads() > 0) { + cassandraStorageSettings.setComposingServiceThreads(nonConfidentialConfiguration.getComposingServiceThreads()); + } manager = new CassandraCradleManager( cassandraConnectionSettings, diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt index f852b8128..ac7c5111d 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt @@ -70,6 +70,7 @@ data class CradleConfidentialConfiguration( data class CradleNonConfidentialConfiguration( var timeout: Long = CassandraStorageSettings.DEFAULT_TIMEOUT, var pageSize: Int = 5000, + var composingServiceThreads: Int = 5, var cradleMaxEventBatchSize: Long = CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE.toLong(), var cradleMaxMessageBatchSize: Long = CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE.toLong(), var prepareStorage: Boolean = false, diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index 3f014fb00..fd9e49605 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -218,6 +218,7 @@ class TestJsonConfiguration { private val CRADLE_NON_CONFIDENTIAL_CONF = CradleNonConfidentialConfiguration( 888, 111, + 5, 123, 321, false, From 8806b6b29843bc22b41059ab2a0b3bb0649d1b32 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 6 Mar 2023 15:47:00 +0400 Subject: [PATCH 103/154] Added counterPersistenceInterval option --- build.gradle | 2 +- .../th2/common/schema/factory/AbstractCommonFactory.java | 3 +++ .../exactpro/th2/common/schema/cradle/CradleConfiguration.kt | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index cf8a519ab..5262821b8 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.0.0-separate-executor-4341698850-6476db6-SNAPSHOT' + cradleVersion = '5.0.0-separate-executor-4342944957-e0e6fe6-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index c43675e77..6639d7a0d 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -544,6 +544,9 @@ public CradleManager getCradleManager() { if (nonConfidentialConfiguration.getComposingServiceThreads() > 0) { cassandraStorageSettings.setComposingServiceThreads(nonConfidentialConfiguration.getComposingServiceThreads()); } + if (nonConfidentialConfiguration.getCounterPersistenceInterval() >= 0) { + cassandraStorageSettings.setCounterPersistenceInterval(nonConfidentialConfiguration.getCounterPersistenceInterval()); + } manager = new CassandraCradleManager( cassandraConnectionSettings, diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt index ac7c5111d..c406c5568 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt @@ -70,11 +70,12 @@ data class CradleConfidentialConfiguration( data class CradleNonConfidentialConfiguration( var timeout: Long = CassandraStorageSettings.DEFAULT_TIMEOUT, var pageSize: Int = 5000, - var composingServiceThreads: Int = 5, + var composingServiceThreads: Int = CradleStorage.DEFAULT_COMPOSING_SERVICE_THREADS, var cradleMaxEventBatchSize: Long = CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE.toLong(), var cradleMaxMessageBatchSize: Long = CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE.toLong(), var prepareStorage: Boolean = false, var statisticsPersistenceIntervalMillis: Long = CassandraStorageSettings.DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS.toLong(), var maxUncompressedMessageBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_MESSAGE_BATCH_SIZE.toLong(), - var maxUncompressedEventBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_TEST_EVENT_SIZE.toLong() + var maxUncompressedEventBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_TEST_EVENT_SIZE.toLong(), + var counterPersistenceInterval: Int = CassandraStorageSettings.DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS, ) : Configuration() \ No newline at end of file From 37fac82f27741d852ee7ab5d3887bdef291fd7db Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 13 Mar 2023 10:13:53 +0400 Subject: [PATCH 104/154] Added FIXMEs afret profiling --- src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt b/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt index 0459db4d6..ed4587256 100644 --- a/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt @@ -72,6 +72,7 @@ data class SessionStats( var rawMessages: Int = 0, ) +//FIXME: com.exactpro.th2.common.metrics.MetricsUtilsKt.incrementTotalMetrics() 30,374 ms (12.5%) fun incrementTotalMetrics( batch: MessageGroupBatch, th2Pin: String, @@ -82,6 +83,7 @@ fun incrementTotalMetrics( val incomingStatsBySession = mutableMapOf() val outgoingStatsBySession = mutableMapOf() + //FIXME: java.util.Collections$UnmodifiableCollection.iterator() 14,357 ms (5.9%) for (group in batch.groupsList) { val messages = group.messagesList From 7c87a9339f980e8c2635f4cbb07ff63322c15127 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 14 Mar 2023 14:30:55 +0400 Subject: [PATCH 105/154] Added storeIndividualMessageSessions option --- build.gradle | 2 +- .../th2/common/schema/factory/AbstractCommonFactory.java | 1 + .../exactpro/th2/common/schema/cradle/CradleConfiguration.kt | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5262821b8..1b7d00f6b 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.0.0-separate-executor-4342944957-e0e6fe6-SNAPSHOT' + cradleVersion = '5.0.0-separate-executor-4414654669-6b36e1b-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 6639d7a0d..3ee8604cd 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -547,6 +547,7 @@ public CradleManager getCradleManager() { if (nonConfidentialConfiguration.getCounterPersistenceInterval() >= 0) { cassandraStorageSettings.setCounterPersistenceInterval(nonConfidentialConfiguration.getCounterPersistenceInterval()); } + cassandraStorageSettings.setStoreIndividualMessageSessions(nonConfidentialConfiguration.getStoreIndividualMessageSessions()); manager = new CassandraCradleManager( cassandraConnectionSettings, diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt index c406c5568..674a404de 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt @@ -17,6 +17,7 @@ package com.exactpro.th2.common.schema.cradle import com.exactpro.cradle.CradleStorage import com.exactpro.cradle.cassandra.CassandraStorageSettings +import com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_STORE_INDIVIDUAL_MESSAGE_SESSIONS import com.exactpro.th2.common.schema.configuration.Configuration import com.fasterxml.jackson.annotation.JsonProperty @@ -78,4 +79,5 @@ data class CradleNonConfidentialConfiguration( var maxUncompressedMessageBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_MESSAGE_BATCH_SIZE.toLong(), var maxUncompressedEventBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_TEST_EVENT_SIZE.toLong(), var counterPersistenceInterval: Int = CassandraStorageSettings.DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS, + var storeIndividualMessageSessions: Boolean = DEFAULT_STORE_INDIVIDUAL_MESSAGE_SESSIONS, ) : Configuration() \ No newline at end of file From 14bdb82cbff73a889ff2c2e1360b308830620fd7 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Fri, 24 Mar 2023 16:15:08 +0400 Subject: [PATCH 106/154] Use CassandraStorageSettings for parse json config --- build.gradle | 3 +- .../schema/factory/AbstractCommonFactory.java | 63 +++++++------------ .../common/schema/factory/CommonFactory.java | 7 ++- .../schema/cradle/CradleConfiguration.kt | 54 +++------------- .../common/schema/TestJsonConfiguration.kt | 62 ++++++++++++++++-- .../cassandra_storage_settings.json | 21 +++++++ .../cradle_non_confidential.json | 2 - .../cradle_non_confidential_combo.json | 27 ++++++++ 8 files changed, 142 insertions(+), 97 deletions(-) create mode 100644 src/test/resources/test_json_configurations/cassandra_storage_settings.json create mode 100644 src/test/resources/test_json_configurations/cradle_non_confidential_combo.json diff --git a/build.gradle b/build.gradle index 1b7d00f6b..c190d627b 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.0.0-separate-executor-4414654669-6b36e1b-SNAPSHOT' + cradleVersion = '5.0.0-separate-executor-4510022000-c224fbb-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } @@ -240,6 +240,7 @@ dependencies { testImplementation 'org.jetbrains.kotlin:kotlin-test-junit5' testImplementation "org.testcontainers:testcontainers:1.17.1" testImplementation "org.testcontainers:rabbitmq:1.17.1" + testImplementation "com.datastax.oss:java-driver-core:4.15.0" testFixturesImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit5', version: kotlin_version testFixturesImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 3ee8604cd..2a70e2e79 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -33,7 +33,6 @@ import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration; import com.exactpro.th2.common.schema.configuration.ConfigurationManager; import com.exactpro.th2.common.schema.cradle.CradleConfidentialConfiguration; -import com.exactpro.th2.common.schema.cradle.CradleConfiguration; import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration; import com.exactpro.th2.common.schema.dictionary.DictionaryType; import com.exactpro.th2.common.schema.exception.CommonFactoryException; @@ -60,6 +59,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.kotlin.KotlinFeature; import com.fasterxml.jackson.module.kotlin.KotlinModule; import io.prometheus.client.exporter.HTTPServer; import io.prometheus.client.hotspot.DefaultExports; @@ -92,8 +92,11 @@ import java.util.jar.Manifest; import java.util.stream.StreamSupport; -import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_CONSISTENCY_LEVEL; -import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_TIMEOUT; +import static com.exactpro.cradle.CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE; +import static com.exactpro.cradle.CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE; +import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS; +import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_TEST_EVENT_SIZE; +import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_RESULT_PAGE_SIZE; import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.StringUtils.defaultIfBlank; @@ -105,7 +108,6 @@ */ public abstract class AbstractCommonFactory implements AutoCloseable { - protected static final String DEFAULT_CRADLE_INSTANCE_NAME = "infra"; protected static final String EXACTPRO_IMPLEMENTATION_VENDOR = "Exactpro Systems LLC"; /** @deprecated please use {@link #LOG4J_PROPERTIES_DEFAULT_PATH} */ @@ -118,7 +120,14 @@ public abstract class AbstractCommonFactory implements AutoCloseable { static { MAPPER.registerModules( - new KotlinModule(), + new KotlinModule.Builder() + .withReflectionCacheSize(512) + .configure(KotlinFeature.NullToEmptyCollection, false) + .configure(KotlinFeature.NullToEmptyMap, false) + .configure(KotlinFeature.NullIsSameAsDefault, false) + .configure(KotlinFeature.SingletonSupport, false) + .configure(KotlinFeature.StrictNullChecks, false) + .build(), new RoutingStrategyModule(MAPPER), new JavaTimeModule() ); @@ -367,7 +376,6 @@ public NotificationRouter getNotificationEventBatchRouter() { /** * Registers custom message router. - * * Unlike the {@link #registerCustomMessageRouter(Class, MessageConverter, Set, Set, String...)} the registered router won't have any additional pins attributes * except {@link QueueAttribute#SUBSCRIBE} for subscribe methods and {@link QueueAttribute#PUBLISH} for send methods * @@ -414,7 +422,6 @@ public void registerCustomMessageRouter( /** * Returns previously registered message router for message of {@code messageClass} type. - * * If the router for that type is not registered yet ,it throws {@link IllegalArgumentException} * @param messageClass custom message class * @param custom message type @@ -482,14 +489,8 @@ protected CradleNonConfidentialConfiguration getCradleNonConfidentialConfigurati return getConfigurationOrLoad(CradleNonConfidentialConfiguration.class, true); } - /** - * @return Schema cradle configuration - * @throws IllegalStateException if cannot read configuration - * @deprecated please use {@link #getCradleManager()} - */ - @Deprecated - public CradleConfiguration getCradleConfiguration() { - return new CradleConfiguration(getCradleConfidentialConfiguration(), getCradleNonConfidentialConfiguration()); + protected CassandraStorageSettings getCassandraStorageSettings() { + return getConfigurationOrLoad(CassandraStorageSettings.class, true); } /** @@ -514,40 +515,24 @@ public CradleManager getCradleManager() { } CradleNonConfidentialConfiguration nonConfidentialConfiguration = getCradleNonConfidentialConfiguration(); - CassandraStorageSettings cassandraStorageSettings = new CassandraStorageSettings( - null, - nonConfidentialConfiguration.getTimeout() > 0 - ? nonConfidentialConfiguration.getTimeout() - : DEFAULT_TIMEOUT, - DEFAULT_CONSISTENCY_LEVEL, - DEFAULT_CONSISTENCY_LEVEL - ); + CassandraStorageSettings cassandraStorageSettings = getCassandraStorageSettings(); cassandraStorageSettings.setKeyspace(confidentialConfiguration.getKeyspace()); - if (nonConfidentialConfiguration.getPageSize() > 0) { + + if (cassandraStorageSettings.getResultPageSize() == DEFAULT_RESULT_PAGE_SIZE && nonConfidentialConfiguration.getPageSize() > 0) { cassandraStorageSettings.setResultPageSize(nonConfidentialConfiguration.getPageSize()); } - if (nonConfidentialConfiguration.getCradleMaxMessageBatchSize() > 0) { + if (cassandraStorageSettings.getMaxMessageBatchSize() == DEFAULT_MAX_MESSAGE_BATCH_SIZE && nonConfidentialConfiguration.getCradleMaxMessageBatchSize() > 0) { cassandraStorageSettings.setMaxMessageBatchSize((int) nonConfidentialConfiguration.getCradleMaxMessageBatchSize()); } - if (nonConfidentialConfiguration.getCradleMaxEventBatchSize() > 0) { + if (cassandraStorageSettings.getMaxTestEventBatchSize() == DEFAULT_MAX_TEST_EVENT_BATCH_SIZE && nonConfidentialConfiguration.getCradleMaxEventBatchSize() > 0) { cassandraStorageSettings.setMaxTestEventBatchSize((int) nonConfidentialConfiguration.getCradleMaxEventBatchSize()); } - if (nonConfidentialConfiguration.getStatisticsPersistenceIntervalMillis() >= 0) { + if (cassandraStorageSettings.getCounterPersistenceInterval() == DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS && nonConfidentialConfiguration.getStatisticsPersistenceIntervalMillis() >= 0) { cassandraStorageSettings.setCounterPersistenceInterval((int) nonConfidentialConfiguration.getStatisticsPersistenceIntervalMillis()); } - if (nonConfidentialConfiguration.getMaxUncompressedMessageBatchSize() > 0) { - cassandraStorageSettings.setMaxUncompressedMessageBatchSize((int) nonConfidentialConfiguration.getMaxUncompressedMessageBatchSize()); - } - if (nonConfidentialConfiguration.getMaxUncompressedEventBatchSize() > 0) { + if (cassandraStorageSettings.getMaxUncompressedTestEventSize() == DEFAULT_MAX_UNCOMPRESSED_TEST_EVENT_SIZE && nonConfidentialConfiguration.getMaxUncompressedEventBatchSize() > 0) { cassandraStorageSettings.setMaxUncompressedTestEventSize((int) nonConfidentialConfiguration.getMaxUncompressedEventBatchSize()); } - if (nonConfidentialConfiguration.getComposingServiceThreads() > 0) { - cassandraStorageSettings.setComposingServiceThreads(nonConfidentialConfiguration.getComposingServiceThreads()); - } - if (nonConfidentialConfiguration.getCounterPersistenceInterval() >= 0) { - cassandraStorageSettings.setCounterPersistenceInterval(nonConfidentialConfiguration.getCounterPersistenceInterval()); - } - cassandraStorageSettings.setStoreIndividualMessageSessions(nonConfidentialConfiguration.getStoreIndividualMessageSessions()); manager = new CassandraCradleManager( cassandraConnectionSettings, @@ -830,7 +815,7 @@ public void close() { prometheusExporter.updateAndGet(server -> { if (server != null) { try { - server.stop(); + server.close(); } catch (Exception e) { LOGGER.error("Failed to close Prometheus exporter", e); } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index f3e3959ad..4374acfbf 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -15,6 +15,7 @@ package com.exactpro.th2.common.schema.factory; +import com.exactpro.cradle.cassandra.CassandraStorageSettings; import com.exactpro.th2.common.grpc.EventBatch; import com.exactpro.th2.common.grpc.MessageBatch; import com.exactpro.th2.common.grpc.MessageGroupBatch; @@ -139,8 +140,7 @@ protected CommonFactory(Class> messageRout @Nullable Path dictionaryTypesDir, @Nullable Path dictionaryAliasesDir, @Nullable Path oldDictionariesDir, - Map environmentVariables, - ConfigurationManager configurationManager) { + Map environmentVariables) { this(new FactorySettings() .messageRouterParsedBatchClass(messageRouterParsedBatchClass) .messageRouterRawBatchClass(messageRouterRawBatchClass) @@ -270,7 +270,7 @@ protected ConfigurationManager getConfigurationManager() { *

* --connectionManagerConfiguration - path to json file with for {@link ConnectionManagerConfiguration} *

- * --cradleManagerConfiguration - path to json file with for {@link CradleNonConfidentialConfiguration} + * --cradleManagerConfiguration - path to json file with for {@link CradleNonConfidentialConfiguration} and {@link CassandraStorageSettings} *

* --namespace - namespace in Kubernetes to find config maps related to the target *

@@ -705,6 +705,7 @@ private static ConfigurationManager createConfigurationManager(FactorySettings s paths.put(GrpcRouterConfiguration.class, defaultPathIfNull(settings.getRouterGRPC(), ROUTER_GRPC_FILE_NAME)); paths.put(CradleConfidentialConfiguration.class, defaultPathIfNull(settings.getCradleConfidential(), CRADLE_CONFIDENTIAL_FILE_NAME)); paths.put(CradleNonConfidentialConfiguration.class, defaultPathIfNull(settings.getCradleNonConfidential(), CRADLE_NON_CONFIDENTIAL_FILE_NAME)); + paths.put(CassandraStorageSettings.class, defaultPathIfNull(settings.getCradleNonConfidential(), CRADLE_NON_CONFIDENTIAL_FILE_NAME)); paths.put(PrometheusConfiguration.class, defaultPathIfNull(settings.getPrometheus(), PROMETHEUS_FILE_NAME)); paths.put(BoxConfiguration.class, defaultPathIfNull(settings.getBoxConfiguration(), BOX_FILE_NAME)); return new ConfigurationManager(paths); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt index 674a404de..17372fedf 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -17,48 +17,10 @@ package com.exactpro.th2.common.schema.cradle import com.exactpro.cradle.CradleStorage import com.exactpro.cradle.cassandra.CassandraStorageSettings -import com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_STORE_INDIVIDUAL_MESSAGE_SESSIONS import com.exactpro.th2.common.schema.configuration.Configuration +import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty -@Deprecated(message = "Please use CradleConfidentialConfiguration and CradleNonConfidentialConfiguration") -data class CradleConfiguration( - var dataCenter: String, - var host: String, - var keyspace: String, - var port: Int, - var username: String?, - var password: String?, - var timeout: Long, - var pageSize: Int, - var cradleMaxEventBatchSize: Long, - var cradleMaxMessageBatchSize: Long, - var prepareStorage: Boolean, - var statisticsPersistenceIntervalMillis: Long, - var maxUncompressedMessageBatchSize: Long, - var maxUncompressedEventBatchSize: Long -) : Configuration() { - constructor( - cradleConfidentialConfiguration: CradleConfidentialConfiguration, - cradleNonConfidentialConfiguration: CradleNonConfidentialConfiguration - ) : this( - cradleConfidentialConfiguration.dataCenter, - cradleConfidentialConfiguration.host, - cradleConfidentialConfiguration.keyspace, - cradleConfidentialConfiguration.port, - cradleConfidentialConfiguration.username, - cradleConfidentialConfiguration.password, - cradleNonConfidentialConfiguration.timeout, - cradleNonConfidentialConfiguration.pageSize, - cradleNonConfidentialConfiguration.cradleMaxEventBatchSize, - cradleNonConfidentialConfiguration.cradleMaxMessageBatchSize, - cradleNonConfidentialConfiguration.prepareStorage, - cradleNonConfidentialConfiguration.statisticsPersistenceIntervalMillis, - cradleNonConfidentialConfiguration.maxUncompressedMessageBatchSize, - cradleNonConfidentialConfiguration.maxUncompressedEventBatchSize - ) -} - data class CradleConfidentialConfiguration( @JsonProperty(required = true) var dataCenter: String, @JsonProperty(required = true) var host: String, @@ -68,16 +30,16 @@ data class CradleConfidentialConfiguration( var password: String? = null ) : Configuration() +@JsonIgnoreProperties(ignoreUnknown = true) data class CradleNonConfidentialConfiguration( - var timeout: Long = CassandraStorageSettings.DEFAULT_TIMEOUT, + var prepareStorage: Boolean = false, + @Deprecated("Please use CassandraStorageSettings.resultPageSize") var pageSize: Int = 5000, - var composingServiceThreads: Int = CradleStorage.DEFAULT_COMPOSING_SERVICE_THREADS, var cradleMaxEventBatchSize: Long = CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE.toLong(), + @Deprecated("Please use CassandraStorageSettings.maxMessageBatchSize") var cradleMaxMessageBatchSize: Long = CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE.toLong(), - var prepareStorage: Boolean = false, + @Deprecated("counterPersistenceInterval") var statisticsPersistenceIntervalMillis: Long = CassandraStorageSettings.DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS.toLong(), - var maxUncompressedMessageBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_MESSAGE_BATCH_SIZE.toLong(), + @Deprecated("maxUncompressedTestEventSize") var maxUncompressedEventBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_TEST_EVENT_SIZE.toLong(), - var counterPersistenceInterval: Int = CassandraStorageSettings.DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS, - var storeIndividualMessageSessions: Boolean = DEFAULT_STORE_INDIVIDUAL_MESSAGE_SESSIONS, ) : Configuration() \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index fd9e49605..1d00385a5 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -15,6 +15,7 @@ package com.exactpro.th2.common.schema +import com.exactpro.cradle.cassandra.CassandraStorageSettings import com.exactpro.th2.common.metrics.PrometheusConfiguration import com.exactpro.th2.common.schema.cradle.CradleConfidentialConfiguration import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration @@ -35,9 +36,12 @@ import com.exactpro.th2.common.schema.strategy.route.impl.RobinRoutingStrategy import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.module.kotlin.KotlinModule import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import org.testcontainers.shaded.org.apache.commons.lang3.builder.EqualsBuilder import java.nio.file.Path class TestJsonConfiguration { @@ -97,11 +101,22 @@ class TestJsonConfiguration { testDeserialize(CRADLE_NON_CONFIDENTIAL_CONF_JSON, CRADLE_NON_CONFIDENTIAL_CONF) } + @Test + fun `test cassandra storage settings json configuration deserialize`() { + assertTrue(EqualsBuilder.reflectionEquals(OBJECT_MAPPER.readValue(CASSANDRA_STORAGE_SETTINGS_JSON, CASSANDRA_STORAGE_SETTINGS::class.java), CASSANDRA_STORAGE_SETTINGS)) + } + @Test fun `test cradle non confidential json configuration serialize and deserialize`() { testSerializeAndDeserialize(CRADLE_NON_CONFIDENTIAL_CONF) } + @Test + fun `test json configuration deserialize combo`() { + assertTrue(EqualsBuilder.reflectionEquals(CASSANDRA_STORAGE_SETTINGS, OBJECT_MAPPER.readValue(CRADLE_NON_CONFIDENTIAL_COMBO_CONF_JSON, CASSANDRA_STORAGE_SETTINGS::class.java))) + assertEquals(CRADLE_NON_CONFIDENTIAL_CONF, OBJECT_MAPPER.readValue(CRADLE_NON_CONFIDENTIAL_COMBO_CONF_JSON, CRADLE_NON_CONFIDENTIAL_CONF::class.java)) + } + @Test fun `test prometheus confidential json configuration deserialize`() { testDeserialize(PROMETHEUS_CONF_JSON, PROMETHEUS_CONF) @@ -214,24 +229,59 @@ class TestJsonConfiguration { "pass" ) + private val CRADLE_NON_CONFIDENTIAL_COMBO_CONF_JSON = loadConfJson("cradle_non_confidential_combo") private val CRADLE_NON_CONFIDENTIAL_CONF_JSON = loadConfJson("cradle_non_confidential") private val CRADLE_NON_CONFIDENTIAL_CONF = CradleNonConfidentialConfiguration( - 888, + false, 111, - 5, 123, 321, - false, 5000, - 1280001, - 1280002 + 1280002, ) + private val CASSANDRA_STORAGE_SETTINGS_JSON = loadConfJson("cassandra_storage_settings") + private val CASSANDRA_STORAGE_SETTINGS = CassandraStorageSettings().apply { +// networkTopologyStrategy = NetworkTopologyStrategyBuilder() +// .add("A", 3) +// .add("B", 3) +// .build() + timeout = 4999 +// writeConsistencyLevel = ConsistencyLevel.THREE +// readConsistencyLevel = ConsistencyLevel.QUORUM + keyspace = "test-keyspace" + keyspaceReplicationFactor = 1 + maxParallelQueries = 1 + resultPageSize = 2 + maxMessageBatchSize = 3 + maxUncompressedMessageBatchSize = 5 + maxTestEventBatchSize = 8 + maxUncompressedTestEventSize = 13 + sessionsCacheSize = 21 + scopesCacheSize = 34 + pageSessionsCacheSize = 55 + pageScopesCacheSize = 89 + sessionStatisticsCacheSize = 144 + pageGroupsCacheSize = 233 + groupsCacheSize = 377 + eventBatchDurationCacheSize = 610 + counterPersistenceInterval = 987 + composingServiceThreads = 1597 + } private val PROMETHEUS_CONF_JSON = loadConfJson("prometheus") private val PROMETHEUS_CONF = PrometheusConfiguration("123.3.3.3", 1234, false) init { - OBJECT_MAPPER.registerModule(KotlinModule()) + OBJECT_MAPPER.registerModule( + KotlinModule.Builder() + .withReflectionCacheSize(512) + .configure(KotlinFeature.NullToEmptyCollection, false) + .configure(KotlinFeature.NullToEmptyMap, false) + .configure(KotlinFeature.NullIsSameAsDefault, false) + .configure(KotlinFeature.SingletonSupport, false) + .configure(KotlinFeature.StrictNullChecks, false) + .build() + ) OBJECT_MAPPER.registerModule(RoutingStrategyModule(OBJECT_MAPPER)) } diff --git a/src/test/resources/test_json_configurations/cassandra_storage_settings.json b/src/test/resources/test_json_configurations/cassandra_storage_settings.json new file mode 100644 index 000000000..873e938c8 --- /dev/null +++ b/src/test/resources/test_json_configurations/cassandra_storage_settings.json @@ -0,0 +1,21 @@ +{ + "timeout": 4999, + "keyspace": "test-keyspace", + "keyspaceReplicationFactor": 1, + "maxParallelQueries": 1, + "resultPageSize": 2, + "maxMessageBatchSize": 3, + "maxUncompressedMessageBatchSize": 5, + "maxTestEventBatchSize": 8, + "maxUncompressedTestEventSize": 13, + "sessionsCacheSize": 21, + "scopesCacheSize": 34, + "pageSessionsCacheSize": 55, + "pageScopesCacheSize": 89, + "sessionStatisticsCacheSize": 144, + "pageGroupsCacheSize": 233, + "groupsCacheSize": 377, + "eventBatchDurationCacheSize": 610, + "counterPersistenceInterval": 987, + "composingServiceThreads": 1597 +} \ No newline at end of file diff --git a/src/test/resources/test_json_configurations/cradle_non_confidential.json b/src/test/resources/test_json_configurations/cradle_non_confidential.json index 874aa01f4..f423c1080 100644 --- a/src/test/resources/test_json_configurations/cradle_non_confidential.json +++ b/src/test/resources/test_json_configurations/cradle_non_confidential.json @@ -1,10 +1,8 @@ { - "timeout": 888, "pageSize": 111, "cradleMaxEventBatchSize": 123, "cradleMaxMessageBatchSize": 321, "prepareStorage": false, "statisticsPersistenceIntervalMillis": 5000, - "maxUncompressedMessageBatchSize": 1280001, "maxUncompressedEventBatchSize": 1280002 } \ No newline at end of file diff --git a/src/test/resources/test_json_configurations/cradle_non_confidential_combo.json b/src/test/resources/test_json_configurations/cradle_non_confidential_combo.json new file mode 100644 index 000000000..e8dbb4098 --- /dev/null +++ b/src/test/resources/test_json_configurations/cradle_non_confidential_combo.json @@ -0,0 +1,27 @@ +{ + "timeout": 4999, + "pageSize": 111, + "cradleMaxEventBatchSize": 123, + "cradleMaxMessageBatchSize": 321, + "prepareStorage": false, + "statisticsPersistenceIntervalMillis": 5000, + "maxUncompressedEventBatchSize": 1280002, + "keyspace": "test-keyspace", + "keyspaceReplicationFactor": 1, + "maxParallelQueries": 1, + "resultPageSize": 2, + "maxMessageBatchSize": 3, + "maxUncompressedMessageBatchSize": 5, + "maxTestEventBatchSize": 8, + "maxUncompressedTestEventSize": 13, + "sessionsCacheSize": 21, + "scopesCacheSize": 34, + "pageSessionsCacheSize": 55, + "pageScopesCacheSize": 89, + "sessionStatisticsCacheSize": 144, + "pageGroupsCacheSize": 233, + "groupsCacheSize": 377, + "eventBatchDurationCacheSize": 610, + "counterPersistenceInterval": 987, + "composingServiceThreads": 1597 +} \ No newline at end of file From 8ba6027859bfb8d9e474deb4921fdcc1b35c3546 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 27 Mar 2023 11:31:05 +0400 Subject: [PATCH 107/154] Use CassandraStorageSettings for parse json config --- .../common/schema/event/EventBatchRouter.java | 2 - .../schema/event/EventBatchSubscriber.java | 3 +- .../schema/factory/AbstractCommonFactory.java | 18 + .../rabbitmq/AbstractRabbitSubscriber.java | 15 - .../th2/common/metrics/CommonMetrics.kt | 1 + .../rabbitmq/custom/RabbitCustomRouter.kt | 5 +- .../schema/message/impl/rabbitmq/demo/Demo.kt | 404 ++++++++++++++++++ .../rabbitmq/demo/DemoMessageBatchRouter.kt | 80 ++++ .../rabbitmq/demo/DemoMessageBatchSender.kt | 63 +++ .../demo/DemoMessageBatchSubscriber.kt | 66 +++ .../group/RabbitMessageGroupBatchRouter.kt | 5 +- .../RabbitMessageGroupBatchSubscriber.kt | 11 +- .../th2/common/event/bean/BaseTest.java | 2 + .../IntegrationTestDemoMessageBatchRouter.kt | 140 ++++++ .../demo/TestDemoMessageBatchRouter.kt | 255 +++++++++++ ...ntegrationTestRabbitMessageBatchRouter.kt} | 0 ...ter.kt => TestRabbitMessageBatchRouter.kt} | 0 17 files changed, 1040 insertions(+), 30 deletions(-) create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoMessageBatchRouter.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoMessageBatchRouter.kt rename src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/{IntegrationTestRabbitMessageGroupBatchRouter.kt => IntegrationTestRabbitMessageBatchRouter.kt} (100%) rename src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/{TestRabbitMessageGroupBatchRouter.kt => TestRabbitMessageBatchRouter.kt} (100%) diff --git a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java index 32ab77768..cd53023c0 100644 --- a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java @@ -21,7 +21,6 @@ import org.jetbrains.annotations.NotNull; import com.exactpro.th2.common.grpc.EventBatch; -import com.exactpro.th2.common.schema.message.FilterFunction; import com.exactpro.th2.common.schema.message.MessageSender; import com.exactpro.th2.common.schema.message.MessageSubscriber; import com.exactpro.th2.common.schema.message.QueueAttribute; @@ -75,7 +74,6 @@ protected MessageSubscriber createSubscriber(QueueConfiguration queu return new EventBatchSubscriber( getConnectionManager(), queueConfiguration.getQueue(), - FilterFunction.DEFAULT_FILTER_FUNCTION, pinName ); } diff --git a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java index b3d01014e..e05e74250 100644 --- a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java @@ -43,10 +43,9 @@ public class EventBatchSubscriber extends AbstractRabbitSubscriber { public EventBatchSubscriber( @NotNull ConnectionManager connectionManager, @NotNull String queue, - @NotNull FilterFunction filterFunc, @NotNull String th2Pin ) { - super(connectionManager, queue, filterFunc, th2Pin, EVENT_TYPE); + super(connectionManager, queue, th2Pin, EVENT_TYPE); } @Override diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 2a70e2e79..f874528b9 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -54,6 +54,8 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.MessageConverter; import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.RabbitCustomRouter; +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatch; +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter; import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule; import com.exactpro.th2.common.schema.util.Log4jConfigUtils; import com.fasterxml.jackson.core.JsonProcessingException; @@ -147,6 +149,7 @@ public abstract class AbstractCommonFactory implements AutoCloseable { private final AtomicReference> messageRouterParsedBatch = new AtomicReference<>(); private final AtomicReference> messageRouterRawBatch = new AtomicReference<>(); private final AtomicReference> messageRouterMessageGroupBatch = new AtomicReference<>(); + private final AtomicReference> demoMessageBatchRouter = new AtomicReference<>(); private final AtomicReference> eventBatchRouter = new AtomicReference<>(); private final AtomicReference> notificationEventBatchRouter = new AtomicReference<>(); private final AtomicReference rootEventId = new AtomicReference<>(); @@ -291,6 +294,21 @@ public MessageRouter getMessageRouterRawBatch() { }); } + /** + * @return Initialized {@link MessageRouter} which works with {@link DemoMessageBatch} + * @throws IllegalStateException if can not read configuration + */ + public MessageRouter getDemoMessageBatchRouter() { + return demoMessageBatchRouter.updateAndGet(router -> { + if (router == null) { + router = new DemoMessageBatchRouter(); + router.init(getMessageRouterContext()); + } + + return router; + }); + } + /** * @return Initialized {@link MessageRouter} which works with {@link MessageGroupBatch} * @throws CommonFactoryException if can not call default constructor from class diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index 38b4782d6..38bb57099 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -22,11 +22,9 @@ import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.exactpro.th2.common.schema.message.MessageSubscriber; import com.exactpro.th2.common.schema.message.SubscriberMonitor; -import com.exactpro.th2.common.schema.message.configuration.RouterFilter; import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import com.google.common.base.Suppliers; -import com.google.protobuf.Message; import com.rabbitmq.client.Delivery; import io.prometheus.client.Counter; import io.prometheus.client.Histogram; @@ -38,7 +36,6 @@ import java.io.IOException; import java.util.Iterator; -import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -82,7 +79,6 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber> consumerMonitor = new AtomicReference<>(emptySupplier()); - private final AtomicReference filterFunc = new AtomicReference<>(); private final String th2Type; private final HealthMetrics healthMetrics = new HealthMetrics(this); @@ -90,13 +86,11 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber filters) { - FilterFunction filterFunction = this.filterFunc.get(); - if (filterFunction == null) { - throw new IllegalStateException("Subscriber is not initialized"); - } - - return filterFunction.apply(message, filters); - } - protected abstract T valueFromBytes(byte[] body) throws Exception; protected abstract String toShortTraceString(T value); diff --git a/src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt b/src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt index a6be78de1..9a51e13c0 100644 --- a/src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt +++ b/src/main/kotlin/com/exactpro/th2/common/metrics/CommonMetrics.kt @@ -34,6 +34,7 @@ const val QUEUE_LABEL = "queue" const val ROUTING_KEY_LABEL = "routing_key" const val BOOK_NAME_LABEL = "book_name" const val SESSION_ALIAS_LABEL = "session_alias" +const val SESSION_GROUP_LABEL = "session_group" const val DIRECTION_LABEL = "direction" const val MESSAGE_TYPE_LABEL = "message_type" diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt index 496a03526..832c64a4c 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt @@ -15,7 +15,6 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.custom -import com.exactpro.th2.common.schema.message.FilterFunction import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.QueueAttribute @@ -75,7 +74,6 @@ class RabbitCustomRouter( return Subscriber( connectionManager, pinConfig.queue, - FilterFunction.DEFAULT_FILTER_FUNCTION, pinName, customTag, converter @@ -105,11 +103,10 @@ class RabbitCustomRouter( private class Subscriber( connectionManager: ConnectionManager, queue: String, - filterFunction: FilterFunction, th2Pin: String, customTag: String, private val converter: MessageConverter - ) : AbstractRabbitSubscriber(connectionManager, queue, filterFunction, th2Pin, customTag) { + ) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, customTag) { override fun valueFromBytes(body: ByteArray): T = converter.fromByteArray(body) override fun toShortTraceString(value: T): String = converter.toTraceString(value) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt new file mode 100644 index 000000000..d993524a0 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt @@ -0,0 +1,404 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo + +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoDirection.INCOMING +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoDirection.OUTGOING +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufUtil +import io.netty.buffer.Unpooled +import java.nio.charset.Charset +import java.time.Instant +import java.util.Objects + +// TODO: maybe make length field a variable length int + +enum class ValueType(val codec: ValueCodec<*>) { + UNKNOWN(UnknownValueCodec), + LONG_TYPE(LongTypeCodec), + STRING_TYPE(StringTypeCodec), + MESSAGE_ID(MessageIdCodec), + BOOK(BookCodec), + SESSION_GROUP(SessionGroupCodec), + SESSION_ALIAS(SessionAliasCodec), + DIRECTION(DirectionCodec), + SEQUENCE(SequenceCodec), + SUBSEQUENCE(SubsequenceCodec), + TIMESTAMP(TimestampCodec), + METADATA(MetadataCodec), + PROTOCOL(ProtocolCodec), + RAW_MESSAGE(RawMessageCodec), + RAW_MESSAGE_BODY(RawMessageBodyCodec), + MESSAGE_BATCH(MessageBatchCodec), + MESSAGE_LIST(MessageListCodec); + + companion object { + private val MAPPING = arrayOfNulls(UByte.MAX_VALUE.toInt()).apply { + ValueType.values().forEach { this[it.codec.type.toInt()] = it } + } + + fun forId(id: UByte): ValueType = MAPPING[id.toInt()] ?: UNKNOWN + } +} + +interface ValueCodec { + val type: UByte + fun encode(source: T, target: ByteBuf) + fun decode(source: ByteBuf): T +} + +object UnknownValueCodec : ValueCodec { + override val type: UByte = 0u + override fun decode(source: ByteBuf): ByteBuf = source.readSlice(source.skipBytes(Byte.SIZE_BYTES).readIntLE()) + override fun encode(source: ByteBuf, target: ByteBuf): Nothing = throw UnsupportedOperationException() +} + +abstract class AbstractCodec(final override val type: UByte) : ValueCodec { + override fun encode(source: T, target: ByteBuf) { + val lengthIndex = target.writeByte(type.toInt()).writerIndex() + target.writeIntLE(0) + val valueIndex = target.writerIndex() + write(target, source) + target.setIntLE(lengthIndex, target.writerIndex() - valueIndex) + } + + protected abstract fun write(buffer: ByteBuf, value: T) + + override fun decode(source: ByteBuf): T { + val tag = source.readByte().toUByte() + check(tag == this.type) { "Unexpected type tag: $tag (expected: ${this.type})" } + val length = source.readIntLE() + return read(source.readSlice(length)) // FIXME: avoid slicing to avoid buffer allocation + } + + protected abstract fun read(buffer: ByteBuf): T +} + +abstract class StringCodec( + type: UByte, + private val charset: Charset = Charsets.UTF_8, +) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): String = buffer.readCharSequence(buffer.readableBytes(), charset).toString() + + override fun write(buffer: ByteBuf, value: String) { + buffer.writeCharSequence(value, charset) + } +} + +abstract class LongCodec(type: UByte) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): Long = buffer.readLongLE() + + override fun write(buffer: ByteBuf, value: Long) { + buffer.writeLongLE(value) + } +} + +abstract class ListCodec(type: UByte, private val elementCodec: ValueCodec) : AbstractCodec>(type) { + override fun read(buffer: ByteBuf): List = mutableListOf().also { list -> + while (buffer.isReadable) { + list += elementCodec.decode(buffer) + } + } + + override fun write(buffer: ByteBuf, value: List) { + value.forEach { elementCodec.encode(it, buffer) } + } +} + +abstract class MapCodec( + type: UByte, + private val keyCodec: ValueCodec, + private val valueCodec: ValueCodec, +) : AbstractCodec>(type) { + override fun read(buffer: ByteBuf): Map = mutableMapOf().apply { + while (buffer.isReadable) { + this[keyCodec.decode(buffer)] = valueCodec.decode(buffer) + } + } + + override fun write(buffer: ByteBuf, value: Map): Unit = value.forEach { (key, value) -> + keyCodec.encode(key, buffer) + valueCodec.encode(value, buffer) + } +} + +abstract class ByteArrayCodec(type: UByte) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): ByteArray = ByteArray(buffer.readableBytes()).apply(buffer::readBytes) + + override fun write(buffer: ByteBuf, value: ByteArray) { + buffer.writeBytes(value) + } +} + +abstract class InstantCodec(type: UByte) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): Instant = Instant.ofEpochSecond(buffer.readLongLE(), buffer.readIntLE().toLong()) + + override fun write(buffer: ByteBuf, value: Instant) { + buffer.writeLongLE(value.epochSecond).writeIntLE(value.nano) + } +} + +object LongTypeCodec : LongCodec(1u) + +object StringTypeCodec : StringCodec(2u) + +object MessageIdCodec : AbstractCodec(10u) { + override fun read(buffer: ByteBuf): DemoMessageId = DemoMessageId().apply { + buffer.forEachValue { codec -> + when (codec) { + is SessionAliasCodec -> sessionAlias = codec.decode(buffer) + is DirectionCodec -> direction = codec.decode(buffer) + is SequenceCodec -> sequence = codec.decode(buffer) + is SubsequenceCodec -> subsequence = codec.decode(buffer) + is TimestampCodec -> timestamp = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: DemoMessageId) { + SessionAliasCodec.encode(value.sessionAlias, buffer) + DirectionCodec.encode(value.direction, buffer) + SequenceCodec.encode(value.sequence, buffer) + SubsequenceCodec.encode(value.subsequence, buffer) + TimestampCodec.encode(value.timestamp, buffer) + } +} + +object BookCodec : StringCodec(101u) + +object SessionGroupCodec : StringCodec(102u) + +object SessionAliasCodec : StringCodec(103u) + +object DirectionCodec : AbstractCodec(104u) { + override fun read(buffer: ByteBuf): DemoDirection = DemoDirection.forId(buffer.readByte().toInt()) + + override fun write(buffer: ByteBuf, value: DemoDirection) { + buffer.writeByte(value.id) + } +} + +object SequenceCodec : LongCodec(105u) + +object SubsequenceCodec : ListCodec(106u, LongTypeCodec) + +object TimestampCodec : InstantCodec(107u) + +object MetadataCodec : MapCodec(11u, StringTypeCodec, StringTypeCodec) + +object ProtocolCodec : StringCodec(12u) + +object RawMessageCodec : AbstractCodec(20u) { + override fun read(buffer: ByteBuf): DemoRawMessage = DemoRawMessage().apply { + buffer.forEachValue { codec -> + when (codec) { + is MessageIdCodec -> id = codec.decode(buffer) + is MetadataCodec -> metadata = codec.decode(buffer) + is ProtocolCodec -> protocol = codec.decode(buffer) + is RawMessageBodyCodec -> body = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: DemoRawMessage) { + MessageIdCodec.encode(value.id, buffer) + MetadataCodec.encode(value.metadata, buffer) + ProtocolCodec.encode(value.protocol, buffer) + RawMessageBodyCodec.encode(value.body, buffer) + } +} + +object RawMessageBodyCodec : ByteArrayCodec(21u) + +object MessageBatchCodec : AbstractCodec(40u) { + override fun read(buffer: ByteBuf): DemoMessageBatch = DemoMessageBatch().apply { + buffer.forEachValue { codec -> + when (codec) { + is BookCodec -> book = codec.decode(buffer) + is SessionGroupCodec -> sessionGroup = codec.decode(buffer) + is MessageListCodec -> messages = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + + messages.forEach { + val id = it.id + id.book = book + id.sessionGroup = sessionGroup + } + } + + override fun write(buffer: ByteBuf, value: DemoMessageBatch) { + BookCodec.encode(value.book, buffer) + SessionGroupCodec.encode(value.sessionGroup, buffer) + MessageListCodec.encode(value.messages, buffer) + } +} + +object MessageListCodec : AbstractCodec>>(41u) { + override fun read(buffer: ByteBuf): List> = mutableListOf>().apply { + buffer.forEachValue { codec -> + when (codec) { + is RawMessageCodec -> this += codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: List>): Unit = value.forEach { message -> + when (message) { + is DemoRawMessage -> RawMessageCodec.encode(message, buffer) + else -> println("Skipping unsupported message type: $message") + } + } +} + +fun ByteBuf.forEachValue(action: (codec: ValueCodec<*>) -> Unit) { + while (isReadable) { + val type = getByte(readerIndex()).toUByte() + + when (val codec = ValueType.forId(type).codec) { + is UnknownValueCodec -> println("Skipping unknown type $type value: ${ByteBufUtil.hexDump(codec.decode(this))}") + else -> action(codec) + } + } +} + +enum class DemoDirection(val id: Int) { + INCOMING(1), + OUTGOING(2); + + companion object { + fun forId(id: Int): DemoDirection = when (id) { + 1 -> INCOMING + 2 -> OUTGOING + else -> error("Unknown direction id: $id") + } + } +} + +data class DemoMessageId( + var book: String = "", + var sessionGroup: String = "", + var sessionAlias: String = "", + var direction: DemoDirection = INCOMING, + var sequence: Long = 0, + var subsequence: List = listOf(), + var timestamp: Instant = Instant.EPOCH, +) { + companion object { + val DEFAULT_INSTANCE: DemoMessageId = DemoMessageId() // FIXME: do smth about its mutability + } +} + +interface DemoMessage { + var id: DemoMessageId + var metadata: Map + var protocol: String + var body: T +} + +data class DemoRawMessage( + override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, + override var metadata: Map = mapOf(), + override var protocol: String = "", + override var body: ByteArray = EMPTY_BODY, +) : DemoMessage { + override fun equals(other: Any?): Boolean = when { + this === other -> true + other !is DemoRawMessage -> false + id != other.id -> false + metadata != other.metadata -> false + protocol != other.protocol -> false + !body.contentEquals(other.body) -> false + else -> true + } + + override fun hashCode(): Int = Objects.hash(id, metadata, protocol, body.contentHashCode()) + + companion object { + private val EMPTY_BODY = ByteArray(0) + } +} + +data class DemoMessageBatch( + var book: String = "", + var sessionGroup: String = "", + var messages: List> = listOf(), +) + +fun DemoMessageBatch.toByteArray() = Unpooled.buffer().run { + MessageBatchCodec.encode(this@toByteArray, this@run) + ByteArray(readableBytes()).apply(::readBytes) +} + +fun main() { + val buffer = Unpooled.buffer() + + val message1 = DemoRawMessage( + id = DemoMessageId( + book = "book1", + sessionGroup = "group1", + sessionAlias = "alias1", + direction = INCOMING, + sequence = 1, + subsequence = listOf(1, 2), + timestamp = Instant.now() + ), + metadata = mapOf( + "prop1" to "value1", + "prop2" to "value2" + ), + protocol = "proto1", + body = byteArrayOf(1, 2, 3, 4) + ) + + val message2 = DemoRawMessage( + id = DemoMessageId( + book = "book1", + sessionGroup = "group1", + sessionAlias = "alias2", + direction = OUTGOING, + sequence = 2, + subsequence = listOf(3, 4), + timestamp = Instant.now() + ), + metadata = mapOf( + "prop3" to "value3", + "prop4" to "value4" + ), + protocol = "proto2", + body = byteArrayOf(5, 6, 7, 8) + ) + + val batch = DemoMessageBatch( + book = "book1", + sessionGroup = "group1", + messages = listOf(message1, message2) + ) + + MessageBatchCodec.encode(batch, buffer) + + val decodedBatch = MessageBatchCodec.decode(buffer) + + println(batch) + println(decodedBatch) + println(batch == decodedBatch) + println(buffer) +} diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt new file mode 100644 index 000000000..458e38a19 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt @@ -0,0 +1,80 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo + +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractFilterStrategy +import com.exactpro.th2.common.schema.filter.strategy.impl.AnyMessageFilterStrategy +import com.exactpro.th2.common.schema.message.MessageSender +import com.exactpro.th2.common.schema.message.MessageSubscriber +import com.exactpro.th2.common.schema.message.QueueAttribute +import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitRouter +import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName +import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName +import com.google.protobuf.Message +import org.jetbrains.annotations.NotNull + +class DemoMessageBatchRouter : AbstractRabbitRouter() { + override fun getDefaultFilterStrategy(): AbstractFilterStrategy { + return AnyMessageFilterStrategy() + } + + override fun splitAndFilter( + message: DemoMessageBatch, + pinConfiguration: @NotNull QueueConfiguration, + pinName: PinName + ): DemoMessageBatch { + //TODO: Implement - whole batch or null + return message + } + + override fun createSender( + pinConfig: QueueConfiguration, + pinName: PinName, + bookName: BookName + ): MessageSender { + return DemoMessageBatchSender( + connectionManager, + pinConfig.exchange, + pinConfig.routingKey, + pinName, + bookName + ) + } + + override fun createSubscriber( + pinConfig: QueueConfiguration, + pinName: PinName + ): MessageSubscriber { + return DemoMessageBatchSubscriber( + connectionManager, + pinConfig.queue, + pinName + ) + } + + override fun DemoMessageBatch.toErrorString(): String = toString() + + override fun getRequiredSendAttributes(): Set = REQUIRED_SEND_ATTRIBUTES + override fun getRequiredSubscribeAttributes(): Set = REQUIRED_SUBSCRIBE_ATTRIBUTES + + companion object { + internal const val DEMO_RAW_MESSAGE_TYPE = "DEMO_RAW_MESSAGE" + internal const val DEMO_RAW_ATTRIBUTE = "demo_raw" + private val REQUIRED_SUBSCRIBE_ATTRIBUTES = setOf(DEMO_RAW_ATTRIBUTE, QueueAttribute.SUBSCRIBE.value) + private val REQUIRED_SEND_ATTRIBUTES = setOf(DEMO_RAW_ATTRIBUTE, QueueAttribute.PUBLISH.value) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt new file mode 100644 index 000000000..6f1b2cd8b --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo + +import com.exactpro.th2.common.metrics.BOOK_NAME_LABEL +import com.exactpro.th2.common.metrics.SESSION_GROUP_LABEL +import com.exactpro.th2.common.metrics.TH2_PIN_LABEL +import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSender +import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter.Companion.DEMO_RAW_MESSAGE_TYPE +import io.prometheus.client.Counter + +class DemoMessageBatchSender( + connectionManager: ConnectionManager, + exchangeName: String, + routingKey: String, + th2Pin: String, + bookName: BookName +) : AbstractRabbitSender( + connectionManager, + exchangeName, + routingKey, + th2Pin, + DEMO_RAW_MESSAGE_TYPE, + bookName +) { + override fun send(value: DemoMessageBatch) { + DEMO_RAW_MESSAGE_PUBLISH_TOTAL + .labels(th2Pin, value.book, value.sessionGroup) + .inc(value.messages.size.toDouble()) + + super.send(value) + } + + override fun valueToBytes(value: DemoMessageBatch): ByteArray = value.toByteArray() + + override fun toShortTraceString(value: DemoMessageBatch): String = value.toString() + + override fun toShortDebugString(value: DemoMessageBatch): String = value.toString() + + companion object { + private val DEMO_RAW_MESSAGE_PUBLISH_TOTAL = Counter.build() + .name("th2_demo_raw_message_publish_total") + .labelNames(TH2_PIN_LABEL, BOOK_NAME_LABEL, SESSION_GROUP_LABEL) + .help("Quantity of published demo raw messages") + .register() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt new file mode 100644 index 000000000..de6aeaf6c --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt @@ -0,0 +1,66 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo + +import com.exactpro.th2.common.metrics.BOOK_NAME_LABEL +import com.exactpro.th2.common.metrics.SESSION_GROUP_LABEL +import com.exactpro.th2.common.metrics.TH2_PIN_LABEL +import com.exactpro.th2.common.schema.message.DeliveryMetadata +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback +import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter.Companion.DEMO_RAW_MESSAGE_TYPE +import com.rabbitmq.client.Delivery +import io.netty.buffer.Unpooled +import io.prometheus.client.Counter + +class DemoMessageBatchSubscriber( + connectionManager: ConnectionManager, + queue: String, + th2Pin: String +) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, DEMO_RAW_MESSAGE_TYPE) { + + override fun valueFromBytes(body: ByteArray): DemoMessageBatch = Unpooled.wrappedBuffer(body).run(MessageBatchCodec::decode) + + override fun toShortTraceString(value: DemoMessageBatch): String = value.toString() + + override fun toShortDebugString(value: DemoMessageBatch): String = value.toString() + + override fun filter(batch: DemoMessageBatch): DemoMessageBatch { + //TODO: Implement - whole batch or null + return batch + } + + override fun handle( + deliveryMetadata: DeliveryMetadata, + delivery: Delivery, + value: DemoMessageBatch, + confirmation: ManualAckDeliveryCallback.Confirmation + ) { + DEMO_RAW_MESSAGE_SUBSCRIBE_TOTAL + .labels(th2Pin, value.book, value.sessionGroup) + .inc(value.messages.size.toDouble()) + super.handle(deliveryMetadata, delivery, value, confirmation) + } + + companion object { + private val DEMO_RAW_MESSAGE_SUBSCRIBE_TOTAL = Counter.build() + .name("th2_demo_raw_message_subscribe_total") + .labelNames(TH2_PIN_LABEL, BOOK_NAME_LABEL, SESSION_GROUP_LABEL) + .help("Quantity of received demo raw messages") + .register() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt index 95183e3c0..54da8c53e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt @@ -17,21 +17,20 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.group import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.metrics.DIRECTION_LABEL +import com.exactpro.th2.common.metrics.MESSAGE_TYPE_LABEL import com.exactpro.th2.common.metrics.SESSION_ALIAS_LABEL import com.exactpro.th2.common.metrics.TH2_PIN_LABEL -import com.exactpro.th2.common.metrics.MESSAGE_TYPE_LABEL import com.exactpro.th2.common.metrics.incrementDroppedMetrics import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractFilterStrategy import com.exactpro.th2.common.schema.filter.strategy.impl.AnyMessageFilterStrategy -import com.exactpro.th2.common.schema.message.FilterFunction import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter -import com.exactpro.th2.common.schema.message.toBuilderWithMetadata import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName +import com.exactpro.th2.common.schema.message.toBuilderWithMetadata import com.google.protobuf.Message import com.google.protobuf.TextFormat import io.prometheus.client.Counter diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt index f2909a91d..d80e80cdc 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt @@ -25,14 +25,15 @@ import com.exactpro.th2.common.metrics.incrementDroppedMetrics import com.exactpro.th2.common.metrics.incrementTotalMetrics import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.FilterFunction +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager -import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback -import com.exactpro.th2.common.schema.message.toBuilderWithMetadata import com.exactpro.th2.common.schema.message.impl.rabbitmq.group.RabbitMessageGroupBatchRouter.Companion.MESSAGE_GROUP_TYPE +import com.exactpro.th2.common.schema.message.toBuilderWithMetadata import com.exactpro.th2.common.schema.message.toShortDebugString import com.google.protobuf.CodedInputStream +import com.google.protobuf.Message import com.rabbitmq.client.Delivery import io.prometheus.client.Counter import io.prometheus.client.Gauge @@ -41,11 +42,11 @@ import mu.KotlinLogging class RabbitMessageGroupBatchSubscriber( connectionManager: ConnectionManager, queue: String, - filterFunction: FilterFunction, + private val filterFunction: FilterFunction, th2Pin: String, private val filters: List, private val messageRecursionLimit: Int -) : AbstractRabbitSubscriber(connectionManager, queue, filterFunction, th2Pin, MESSAGE_GROUP_TYPE) { +) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, MESSAGE_GROUP_TYPE) { private val logger = KotlinLogging.logger {} override fun valueFromBytes(body: ByteArray): MessageGroupBatch = parseEncodedBatch(body) @@ -96,6 +97,8 @@ class RabbitMessageGroupBatchSubscriber( super.handle(deliveryMetadata, delivery, value, confirmation) } + private fun callFilterFunction(message: Message, filters: List): Boolean = filterFunction.apply(message, filters) + private fun parseEncodedBatch(body: ByteArray?): MessageGroupBatch { val ins = CodedInputStream.newInstance(body) ins.setRecursionLimit(messageRecursionLimit) diff --git a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java index ef7cbfcf8..7f731f52c 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java @@ -31,6 +31,8 @@ public class BaseTest { public static final BoxConfiguration BOX_CONFIGURATION = new BoxConfiguration(); public static final String BOOK_NAME = BOX_CONFIGURATION.getBookName(); + public static final String SESSION_GROUP = "test-group"; + public static final String SESSION_ALIAS = "test-alias"; public static final EventID PARENT_EVENT_ID = toEventID(Instant.now(), BOOK_NAME, "id"); private static final ObjectMapper jacksonMapper = jacksonObjectMapper(); diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoMessageBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoMessageBatchRouter.kt new file mode 100644 index 000000000..d70e0fa2a --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoMessageBatchRouter.kt @@ -0,0 +1,140 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo + +import com.exactpro.th2.common.annotations.IntegrationTest +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration +import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration +import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.rabbitmq.client.BuiltinExchangeType +import mu.KotlinLogging +import org.junit.jupiter.api.Test +import org.mockito.kotlin.mock +import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.test.assertTrue + +@IntegrationTest +class IntegrationTestDemoMessageBatchRouter { + + @Test + fun `subscribe to exclusive queue`() { + RabbitMQContainer(DockerImageName.parse(RABBITMQ_3_8_MANAGEMENT_ALPINE)) + .use { rabbitMQContainer -> + rabbitMQContainer.start() + LOGGER.info { "Started with port ${rabbitMQContainer.amqpPort}" } + + createConnectionManager(rabbitMQContainer).use { firstManager -> + createRouter(firstManager).use { firstRouter -> + createConnectionManager(rabbitMQContainer).use { secondManager -> + createRouter(secondManager).use { secondRouter -> + val counter = CountDownLatch(1) + val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } + try { + secondRouter.sendExclusive(monitor.queue, DemoMessageBatch()) + assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } + + } finally { + monitor.unsubscribe() + } + } + } + } + } + } + } + + @Test + fun `send receive message group batch`() { + RabbitMQContainer(DockerImageName.parse(RABBITMQ_3_8_MANAGEMENT_ALPINE)) + .withExchange(EXCHANGE, BuiltinExchangeType.DIRECT.type, false, false, true, emptyMap()) + .withQueue(QUEUE_NAME) + .withBinding(EXCHANGE, QUEUE_NAME, emptyMap(), ROUTING_KEY, "queue") + .use { rabbitMQContainer -> + rabbitMQContainer.start() + LOGGER.info { "Started with port ${rabbitMQContainer.amqpPort}" } + + createConnectionManager(rabbitMQContainer).use { firstManager -> + createRouter(firstManager).use { firstRouter -> + createConnectionManager(rabbitMQContainer).use { secondManager -> + createRouter(secondManager).use { secondRouter -> + val counter = CountDownLatch(1) + val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } + try { + + secondRouter.sendExclusive(monitor.queue, DemoMessageBatch()) + assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } + + } finally { + monitor.unsubscribe() + } + } + } + } + } + } + } + + private fun createRouter(connectionManager: ConnectionManager) = DemoMessageBatchRouter() + .apply { + init( + DefaultMessageRouterContext( + connectionManager, + mock { }, + MessageRouterConfiguration(), + BoxConfiguration() + ) + ) + } + + private fun createConnectionManager( + rabbitMQContainer: RabbitMQContainer, + prefetchCount: Int = DEFAULT_PREFETCH_COUNT, + confirmationTimeout: Duration = DEFAULT_CONFIRMATION_TIMEOUT + ) = ConnectionManager( + RabbitMQConfiguration( + host = rabbitMQContainer.host, + vHost = "", + port = rabbitMQContainer.amqpPort, + username = rabbitMQContainer.adminUsername, + password = rabbitMQContainer.adminPassword, + ), + ConnectionManagerConfiguration( + subscriberName = "test", + prefetchCount = prefetchCount, + confirmationTimeout = confirmationTimeout, + ), + ) { + LOGGER.error { "Fatal connection problem" } + } + + companion object { + private val LOGGER = KotlinLogging.logger { } + + private const val RABBITMQ_3_8_MANAGEMENT_ALPINE = "rabbitmq:3.8-management-alpine" + private const val ROUTING_KEY = "routingKey" + private const val QUEUE_NAME = "queue" + private const val EXCHANGE = "test-exchange" + + private const val DEFAULT_PREFETCH_COUNT = 10 + private val DEFAULT_CONFIRMATION_TIMEOUT: Duration = Duration.ofSeconds(1) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoMessageBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoMessageBatchRouter.kt new file mode 100644 index 000000000..c47e039e7 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoMessageBatchRouter.kt @@ -0,0 +1,255 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo + +import com.exactpro.th2.common.event.bean.BaseTest.BOOK_NAME +import com.exactpro.th2.common.event.bean.BaseTest.BOX_CONFIGURATION +import com.exactpro.th2.common.event.bean.BaseTest.SESSION_ALIAS +import com.exactpro.th2.common.event.bean.BaseTest.SESSION_GROUP +import com.exactpro.th2.common.grpc.MessageGroupBatch +import com.exactpro.th2.common.schema.message.ExclusiveSubscriberMonitor +import com.exactpro.th2.common.schema.message.MessageRouter +import com.exactpro.th2.common.schema.message.configuration.GlobalNotificationConfiguration +import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration +import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration +import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter.Companion.DEMO_RAW_ATTRIBUTE +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.function.Executable +import org.mockito.kotlin.any +import org.mockito.kotlin.anyOrNull +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify + +class TestDemoMessageBatchRouter { + private val connectionConfiguration = ConnectionManagerConfiguration() + private val monitor: ExclusiveSubscriberMonitor = mock { } + private val connectionManager: ConnectionManager = mock { + on { configuration }.thenReturn(connectionConfiguration) + on { basicConsume(any(), any(), any()) }.thenReturn(monitor) + } + + @Nested + inner class Publishing { + private val router = createRouter(mapOf( + "test-pine" to QueueConfiguration( + routingKey = "", + queue = "subscribe", + exchange = "test-exchange", + attributes = listOf("subscribe", DEMO_RAW_ATTRIBUTE) + ), + "test-pin1" to QueueConfiguration( + routingKey = "test", + queue = "", + exchange = "test-exchange", + attributes = listOf("publish", DEMO_RAW_ATTRIBUTE), + ), + "test-pin2" to QueueConfiguration( + routingKey = "test2", + queue = "", + exchange = "test-exchange", + attributes = listOf("publish", "test", DEMO_RAW_ATTRIBUTE), + ) + )) + + @Test + fun `publishes to the correct pin according to attributes`() { + val batch = DemoMessageBatch( + BOOK_NAME, + SESSION_GROUP, + listOf( + DemoRawMessage( + DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), + body = byteArrayOf( 1, 2, 3 ) + ) + ) + ) + router.send(batch, "test") + + + val captor = argumentCaptor() + verify(connectionManager).basicPublish(eq("test-exchange"), eq("test2"), anyOrNull(), captor.capture()) + val publishedBytes = captor.firstValue + assertArrayEquals(batch.toByteArray(), publishedBytes) { + "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" + } + } + + @Test + fun `reports about extra pins matches the publication`() { + Assertions.assertThrows(IllegalStateException::class.java) { + router.send(DEMO_MESSAGE_BATCH, DEMO_RAW_ATTRIBUTE) + }.apply { + Assertions.assertEquals( + "Found incorrect number of pins [test-pin1, test-pin2] to the send operation by attributes [demo_raw, publish] and filters, expected 1, actual 2", + message + ) + } + } + + @Test + fun `reports about no pins matches the publication`() { + Assertions.assertThrows(IllegalStateException::class.java) { + router.send(DEMO_MESSAGE_BATCH, DEMO_RAW_ATTRIBUTE, "unexpected") + }.apply { + Assertions.assertEquals( + "Found incorrect number of pins [] to the send operation by attributes [demo_raw, unexpected, publish] and filters, expected 1, actual 0", + message + ) + } + } + + @Test + fun `publishes to all correct pin according to attributes`() { + router.sendAll(DEMO_MESSAGE_BATCH) + + val captor = argumentCaptor() + verify(connectionManager).basicPublish(eq("test-exchange"), eq("test"), anyOrNull(), captor.capture()) + verify(connectionManager).basicPublish(eq("test-exchange"), eq("test2"), anyOrNull(), captor.capture()) + val originalBytes = DEMO_MESSAGE_BATCH.toByteArray() + Assertions.assertAll( + Executable { + val publishedBytes = captor.firstValue + assertArrayEquals(originalBytes, publishedBytes) { + "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" + } + }, + Executable { + val publishedBytes = captor.secondValue + assertArrayEquals(originalBytes, publishedBytes) { + "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" + } + } + ) + } + } + + @Nested + inner class Subscribing { + private val router = createRouter( + mapOf( + "test" to QueueConfiguration( + routingKey = "publish", + queue = "", + exchange = "test-exchange", + attributes = listOf("publish", DEMO_RAW_ATTRIBUTE, "test") + ), + "test1" to QueueConfiguration( + routingKey = "", + queue = "queue1", + exchange = "test-exchange", + attributes = listOf("subscribe", DEMO_RAW_ATTRIBUTE, "1") + ), + "test2" to QueueConfiguration( + routingKey = "", + queue = "queue2", + exchange = "test-exchange", + attributes = listOf("subscribe", DEMO_RAW_ATTRIBUTE, "2") + ) + ) + ) + + @Test + fun `subscribes to correct queue`() { + val monitor = router.subscribe(mock { }, "1", DEMO_RAW_ATTRIBUTE) + Assertions.assertNotNull(monitor) { "monitor must not be null" } + + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + } + @Test + fun `subscribes with manual ack to correct queue`() { + val monitor = router.subscribeWithManualAck(mock { }, "1", DEMO_RAW_ATTRIBUTE) + Assertions.assertNotNull(monitor) { "monitor must not be null" } + + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + } + + @Test + fun `subscribes to all matched queues`() { + val monitor = router.subscribeAll(mock { }, DEMO_RAW_ATTRIBUTE) + Assertions.assertNotNull(monitor) { "monitor must not be null" } + + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + verify(connectionManager).basicConsume(eq("queue2"), any(), any()) + } + + @Test + fun `reports if more that one queue matches`() { + Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }) } + .apply { + Assertions.assertEquals( + "Found incorrect number of pins [test1, test2] to subscribe operation by attributes [demo_raw, subscribe] and filters, expected 1, actual 2", + message + ) + } + } + + @Test + fun `reports if no queue matches`() { + Assertions.assertAll( + Executable { + Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }, DEMO_RAW_ATTRIBUTE, "unexpected") } + .apply { + Assertions.assertEquals( + "Found incorrect number of pins [] to subscribe operation by attributes [demo_raw, unexpected, subscribe] and filters, expected 1, actual 0", + message + ) + } + }, + Executable { + Assertions.assertThrows(IllegalStateException::class.java) { router.subscribeAll(mock { }, DEMO_RAW_ATTRIBUTE, "unexpected") } + .apply { + Assertions.assertEquals( + "Found incorrect number of pins [] to subscribe all operation by attributes [demo_raw, unexpected, subscribe] and filters, expected 1 or more, actual 0", + message + ) + } + } + ) + } + } + + private fun createRouter(pins: Map): MessageRouter = + DemoMessageBatchRouter().apply { + init(DefaultMessageRouterContext( + connectionManager, + mock { }, + MessageRouterConfiguration(pins, GlobalNotificationConfiguration()), + BOX_CONFIGURATION + )) + } + + companion object { + private val DEMO_MESSAGE_BATCH = DemoMessageBatch( + BOOK_NAME, + SESSION_GROUP, + listOf( + DemoRawMessage( + DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), + body = byteArrayOf(1, 2, 3) + ) + ) + ) + } +} diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageBatchRouter.kt similarity index 100% rename from src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageGroupBatchRouter.kt rename to src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/IntegrationTestRabbitMessageBatchRouter.kt diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageBatchRouter.kt similarity index 100% rename from src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageGroupBatchRouter.kt rename to src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageBatchRouter.kt From 1f1de4569602a032eb8bc4ae651f5edb1c9828a0 Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Tue, 28 Mar 2023 11:10:13 +0400 Subject: [PATCH 108/154] Add groups to new interop protocol --- .../schema/factory/AbstractCommonFactory.java | 10 +-- .../schema/message/impl/rabbitmq/demo/Demo.kt | 74 +++++++++++------ .../rabbitmq/demo/DemoMessageBatchRouter.kt | 18 ++-- .../rabbitmq/demo/DemoMessageBatchSender.kt | 14 ++-- .../demo/DemoMessageBatchSubscriber.kt | 18 ++-- ...=> IntegrationTestDemoGroupBatchRouter.kt} | 6 +- ...hRouter.kt => TestDemoGroupBatchRouter.kt} | 82 +++++++++++-------- 7 files changed, 129 insertions(+), 93 deletions(-) rename src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/{IntegrationTestDemoMessageBatchRouter.kt => IntegrationTestDemoGroupBatchRouter.kt} (98%) rename src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/{TestDemoMessageBatchRouter.kt => TestDemoGroupBatchRouter.kt} (82%) diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index f874528b9..36900ff01 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -54,7 +54,7 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.MessageConverter; import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.RabbitCustomRouter; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatch; +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoGroupBatch; import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter; import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule; import com.exactpro.th2.common.schema.util.Log4jConfigUtils; @@ -149,7 +149,7 @@ public abstract class AbstractCommonFactory implements AutoCloseable { private final AtomicReference> messageRouterParsedBatch = new AtomicReference<>(); private final AtomicReference> messageRouterRawBatch = new AtomicReference<>(); private final AtomicReference> messageRouterMessageGroupBatch = new AtomicReference<>(); - private final AtomicReference> demoMessageBatchRouter = new AtomicReference<>(); + private final AtomicReference> demoMessageBatchRouter = new AtomicReference<>(); private final AtomicReference> eventBatchRouter = new AtomicReference<>(); private final AtomicReference> notificationEventBatchRouter = new AtomicReference<>(); private final AtomicReference rootEventId = new AtomicReference<>(); @@ -295,10 +295,10 @@ public MessageRouter getMessageRouterRawBatch() { } /** - * @return Initialized {@link MessageRouter} which works with {@link DemoMessageBatch} - * @throws IllegalStateException if can not read configuration + * @return Initialized {@link MessageRouter} which works with {@link DemoGroupBatch} + * @throws IllegalStateException if can not read configuration */ - public MessageRouter getDemoMessageBatchRouter() { + public MessageRouter getDemoMessageBatchRouter() { return demoMessageBatchRouter.updateAndGet(router -> { if (router == null) { router = new DemoMessageBatchRouter(); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt index d993524a0..b3c8495d8 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt @@ -43,8 +43,10 @@ enum class ValueType(val codec: ValueCodec<*>) { PROTOCOL(ProtocolCodec), RAW_MESSAGE(RawMessageCodec), RAW_MESSAGE_BODY(RawMessageBodyCodec), - MESSAGE_BATCH(MessageBatchCodec), - MESSAGE_LIST(MessageListCodec); + MESSAGE_GROUP(MessageGroupCodec), + MESSAGE_LIST(MessageListCodec), + GROUP_BATCH(GroupBatchCodec), + GROUP_LIST(GroupListCodec); companion object { private val MAPPING = arrayOfNulls(UByte.MAX_VALUE.toInt()).apply { @@ -226,27 +228,17 @@ object RawMessageCodec : AbstractCodec(20u) { object RawMessageBodyCodec : ByteArrayCodec(21u) -object MessageBatchCodec : AbstractCodec(40u) { - override fun read(buffer: ByteBuf): DemoMessageBatch = DemoMessageBatch().apply { +object MessageGroupCodec : AbstractCodec(40u) { + override fun read(buffer: ByteBuf): DemoMessageGroup = DemoMessageGroup().apply { buffer.forEachValue { codec -> when (codec) { - is BookCodec -> book = codec.decode(buffer) - is SessionGroupCodec -> sessionGroup = codec.decode(buffer) is MessageListCodec -> messages = codec.decode(buffer) else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") } } - - messages.forEach { - val id = it.id - id.book = book - id.sessionGroup = sessionGroup - } } - override fun write(buffer: ByteBuf, value: DemoMessageBatch) { - BookCodec.encode(value.book, buffer) - SessionGroupCodec.encode(value.sessionGroup, buffer) + override fun write(buffer: ByteBuf, value: DemoMessageGroup) { MessageListCodec.encode(value.messages, buffer) } } @@ -269,6 +261,35 @@ object MessageListCodec : AbstractCodec>>(41u) { } } +object GroupBatchCodec : AbstractCodec(50u) { + override fun read(buffer: ByteBuf): DemoGroupBatch = DemoGroupBatch().apply { + buffer.forEachValue { codec -> + when (codec) { + is BookCodec -> book = codec.decode(buffer) + is SessionGroupCodec -> sessionGroup = codec.decode(buffer) + is GroupListCodec -> groups = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + + groups.forEach { + it.messages.forEach { + val id = it.id + id.book = book + id.sessionGroup = sessionGroup + } + } + } + + override fun write(buffer: ByteBuf, value: DemoGroupBatch) { + BookCodec.encode(value.book, buffer) + SessionGroupCodec.encode(value.sessionGroup, buffer) + GroupListCodec.encode(value.groups, buffer) + } +} + +object GroupListCodec : ListCodec(51u, MessageGroupCodec) + fun ByteBuf.forEachValue(action: (codec: ValueCodec<*>) -> Unit) { while (isReadable) { val type = getByte(readerIndex()).toUByte() @@ -326,8 +347,7 @@ data class DemoRawMessage( id != other.id -> false metadata != other.metadata -> false protocol != other.protocol -> false - !body.contentEquals(other.body) -> false - else -> true + else -> body.contentEquals(other.body) } override fun hashCode(): Int = Objects.hash(id, metadata, protocol, body.contentHashCode()) @@ -337,14 +357,18 @@ data class DemoRawMessage( } } -data class DemoMessageBatch( +data class DemoMessageGroup( + var messages: List> = listOf(), +) + +data class DemoGroupBatch( var book: String = "", var sessionGroup: String = "", - var messages: List> = listOf(), + var groups: List = listOf(), ) -fun DemoMessageBatch.toByteArray() = Unpooled.buffer().run { - MessageBatchCodec.encode(this@toByteArray, this@run) +fun DemoGroupBatch.toByteArray() = Unpooled.buffer().run { + GroupBatchCodec.encode(this@toByteArray, this@run) ByteArray(readableBytes()).apply(::readBytes) } @@ -387,15 +411,15 @@ fun main() { body = byteArrayOf(5, 6, 7, 8) ) - val batch = DemoMessageBatch( + val batch = DemoGroupBatch( book = "book1", sessionGroup = "group1", - messages = listOf(message1, message2) + groups = listOf(DemoMessageGroup(listOf(message1, message2))) ) - MessageBatchCodec.encode(batch, buffer) + GroupBatchCodec.encode(batch, buffer) - val decodedBatch = MessageBatchCodec.decode(buffer) + val decodedBatch = GroupBatchCodec.decode(buffer) println(batch) println(decodedBatch) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt index 458e38a19..e993e5c74 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt @@ -27,16 +27,16 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName import com.google.protobuf.Message import org.jetbrains.annotations.NotNull -class DemoMessageBatchRouter : AbstractRabbitRouter() { +class DemoMessageBatchRouter : AbstractRabbitRouter() { override fun getDefaultFilterStrategy(): AbstractFilterStrategy { return AnyMessageFilterStrategy() } override fun splitAndFilter( - message: DemoMessageBatch, + message: DemoGroupBatch, pinConfiguration: @NotNull QueueConfiguration, - pinName: PinName - ): DemoMessageBatch { + pinName: PinName, + ): DemoGroupBatch { //TODO: Implement - whole batch or null return message } @@ -44,8 +44,8 @@ class DemoMessageBatchRouter : AbstractRabbitRouter() { override fun createSender( pinConfig: QueueConfiguration, pinName: PinName, - bookName: BookName - ): MessageSender { + bookName: BookName, + ): MessageSender { return DemoMessageBatchSender( connectionManager, pinConfig.exchange, @@ -57,8 +57,8 @@ class DemoMessageBatchRouter : AbstractRabbitRouter() { override fun createSubscriber( pinConfig: QueueConfiguration, - pinName: PinName - ): MessageSubscriber { + pinName: PinName, + ): MessageSubscriber { return DemoMessageBatchSubscriber( connectionManager, pinConfig.queue, @@ -66,7 +66,7 @@ class DemoMessageBatchRouter : AbstractRabbitRouter() { ) } - override fun DemoMessageBatch.toErrorString(): String = toString() + override fun DemoGroupBatch.toErrorString(): String = toString() override fun getRequiredSendAttributes(): Set = REQUIRED_SEND_ATTRIBUTES override fun getRequiredSubscribeAttributes(): Set = REQUIRED_SUBSCRIBE_ATTRIBUTES diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt index 6f1b2cd8b..48feb27d4 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt @@ -30,8 +30,8 @@ class DemoMessageBatchSender( exchangeName: String, routingKey: String, th2Pin: String, - bookName: BookName -) : AbstractRabbitSender( + bookName: BookName, +) : AbstractRabbitSender( connectionManager, exchangeName, routingKey, @@ -39,19 +39,19 @@ class DemoMessageBatchSender( DEMO_RAW_MESSAGE_TYPE, bookName ) { - override fun send(value: DemoMessageBatch) { + override fun send(value: DemoGroupBatch) { DEMO_RAW_MESSAGE_PUBLISH_TOTAL .labels(th2Pin, value.book, value.sessionGroup) - .inc(value.messages.size.toDouble()) + .inc(value.groups.size.toDouble()) super.send(value) } - override fun valueToBytes(value: DemoMessageBatch): ByteArray = value.toByteArray() + override fun valueToBytes(value: DemoGroupBatch): ByteArray = value.toByteArray() - override fun toShortTraceString(value: DemoMessageBatch): String = value.toString() + override fun toShortTraceString(value: DemoGroupBatch): String = value.toString() - override fun toShortDebugString(value: DemoMessageBatch): String = value.toString() + override fun toShortDebugString(value: DemoGroupBatch): String = value.toString() companion object { private val DEMO_RAW_MESSAGE_PUBLISH_TOTAL = Counter.build() diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt index de6aeaf6c..0963a7a05 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt @@ -30,16 +30,16 @@ import io.prometheus.client.Counter class DemoMessageBatchSubscriber( connectionManager: ConnectionManager, queue: String, - th2Pin: String -) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, DEMO_RAW_MESSAGE_TYPE) { + th2Pin: String, +) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, DEMO_RAW_MESSAGE_TYPE) { - override fun valueFromBytes(body: ByteArray): DemoMessageBatch = Unpooled.wrappedBuffer(body).run(MessageBatchCodec::decode) + override fun valueFromBytes(body: ByteArray): DemoGroupBatch = Unpooled.wrappedBuffer(body).run(GroupBatchCodec::decode) - override fun toShortTraceString(value: DemoMessageBatch): String = value.toString() + override fun toShortTraceString(value: DemoGroupBatch): String = value.toString() - override fun toShortDebugString(value: DemoMessageBatch): String = value.toString() + override fun toShortDebugString(value: DemoGroupBatch): String = value.toString() - override fun filter(batch: DemoMessageBatch): DemoMessageBatch { + override fun filter(batch: DemoGroupBatch): DemoGroupBatch { //TODO: Implement - whole batch or null return batch } @@ -47,12 +47,12 @@ class DemoMessageBatchSubscriber( override fun handle( deliveryMetadata: DeliveryMetadata, delivery: Delivery, - value: DemoMessageBatch, - confirmation: ManualAckDeliveryCallback.Confirmation + value: DemoGroupBatch, + confirmation: ManualAckDeliveryCallback.Confirmation, ) { DEMO_RAW_MESSAGE_SUBSCRIBE_TOTAL .labels(th2Pin, value.book, value.sessionGroup) - .inc(value.messages.size.toDouble()) + .inc(value.groups.size.toDouble()) super.handle(deliveryMetadata, delivery, value, confirmation) } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoMessageBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoGroupBatchRouter.kt similarity index 98% rename from src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoMessageBatchRouter.kt rename to src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoGroupBatchRouter.kt index d70e0fa2a..1cd87b985 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoMessageBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoGroupBatchRouter.kt @@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit import kotlin.test.assertTrue @IntegrationTest -class IntegrationTestDemoMessageBatchRouter { +class IntegrationTestDemoGroupBatchRouter { @Test fun `subscribe to exclusive queue`() { @@ -49,7 +49,7 @@ class IntegrationTestDemoMessageBatchRouter { val counter = CountDownLatch(1) val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } try { - secondRouter.sendExclusive(monitor.queue, DemoMessageBatch()) + secondRouter.sendExclusive(monitor.queue, DemoGroupBatch()) assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } } finally { @@ -80,7 +80,7 @@ class IntegrationTestDemoMessageBatchRouter { val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } try { - secondRouter.sendExclusive(monitor.queue, DemoMessageBatch()) + secondRouter.sendExclusive(monitor.queue, DemoGroupBatch()) assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } } finally { diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoMessageBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt similarity index 82% rename from src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoMessageBatchRouter.kt rename to src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt index c47e039e7..d82c18c98 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoMessageBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt @@ -42,7 +42,7 @@ import org.mockito.kotlin.eq import org.mockito.kotlin.mock import org.mockito.kotlin.verify -class TestDemoMessageBatchRouter { +class TestDemoGroupBatchRouter { private val connectionConfiguration = ConnectionManagerConfiguration() private val monitor: ExclusiveSubscriberMonitor = mock { } private val connectionManager: ConnectionManager = mock { @@ -52,36 +52,42 @@ class TestDemoMessageBatchRouter { @Nested inner class Publishing { - private val router = createRouter(mapOf( - "test-pine" to QueueConfiguration( - routingKey = "", - queue = "subscribe", - exchange = "test-exchange", - attributes = listOf("subscribe", DEMO_RAW_ATTRIBUTE) - ), - "test-pin1" to QueueConfiguration( - routingKey = "test", - queue = "", - exchange = "test-exchange", - attributes = listOf("publish", DEMO_RAW_ATTRIBUTE), - ), - "test-pin2" to QueueConfiguration( - routingKey = "test2", - queue = "", - exchange = "test-exchange", - attributes = listOf("publish", "test", DEMO_RAW_ATTRIBUTE), + private val router = createRouter( + mapOf( + "test-pine" to QueueConfiguration( + routingKey = "", + queue = "subscribe", + exchange = "test-exchange", + attributes = listOf("subscribe", DEMO_RAW_ATTRIBUTE) + ), + "test-pin1" to QueueConfiguration( + routingKey = "test", + queue = "", + exchange = "test-exchange", + attributes = listOf("publish", DEMO_RAW_ATTRIBUTE), + ), + "test-pin2" to QueueConfiguration( + routingKey = "test2", + queue = "", + exchange = "test-exchange", + attributes = listOf("publish", "test", DEMO_RAW_ATTRIBUTE), + ) ) - )) + ) @Test fun `publishes to the correct pin according to attributes`() { - val batch = DemoMessageBatch( + val batch = DemoGroupBatch( BOOK_NAME, SESSION_GROUP, listOf( - DemoRawMessage( - DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), - body = byteArrayOf( 1, 2, 3 ) + DemoMessageGroup( + listOf( + DemoRawMessage( + DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), + body = byteArrayOf(1, 2, 3) + ) + ) ) ) ) @@ -230,24 +236,30 @@ class TestDemoMessageBatchRouter { } } - private fun createRouter(pins: Map): MessageRouter = + private fun createRouter(pins: Map): MessageRouter = DemoMessageBatchRouter().apply { - init(DefaultMessageRouterContext( - connectionManager, - mock { }, - MessageRouterConfiguration(pins, GlobalNotificationConfiguration()), - BOX_CONFIGURATION - )) + init( + DefaultMessageRouterContext( + connectionManager, + mock { }, + MessageRouterConfiguration(pins, GlobalNotificationConfiguration()), + BOX_CONFIGURATION + ) + ) } companion object { - private val DEMO_MESSAGE_BATCH = DemoMessageBatch( + private val DEMO_MESSAGE_BATCH = DemoGroupBatch( BOOK_NAME, SESSION_GROUP, listOf( - DemoRawMessage( - DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), - body = byteArrayOf(1, 2, 3) + DemoMessageGroup( + listOf( + DemoRawMessage( + DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), + body = byteArrayOf(1, 2, 3) + ) + ) ) ) ) From 0b4ad85cab5d62541555dfd32edfcb5779780cf9 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 28 Mar 2023 11:37:08 +0400 Subject: [PATCH 109/154] Removed demo_raw attribute for DemoMessageBatchRouter --- build.gradle | 2 +- .../rabbitmq/demo/DemoMessageBatchRouter.kt | 6 +-- .../rabbitmq/demo/TestDemoGroupBatchRouter.kt | 37 +++++++++---------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/build.gradle b/build.gradle index c190d627b..53c0a6644 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.0.0-separate-executor-4510022000-c224fbb-SNAPSHOT' + cradleVersion = '5.0.0-separate-executor-4540242539-6c92e55-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt index e993e5c74..3bc43d152 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt @@ -73,8 +73,8 @@ class DemoMessageBatchRouter : AbstractRabbitRouter() { companion object { internal const val DEMO_RAW_MESSAGE_TYPE = "DEMO_RAW_MESSAGE" - internal const val DEMO_RAW_ATTRIBUTE = "demo_raw" - private val REQUIRED_SUBSCRIBE_ATTRIBUTES = setOf(DEMO_RAW_ATTRIBUTE, QueueAttribute.SUBSCRIBE.value) - private val REQUIRED_SEND_ATTRIBUTES = setOf(DEMO_RAW_ATTRIBUTE, QueueAttribute.PUBLISH.value) +// internal const val DEMO_RAW_ATTRIBUTE = "demo_raw" // It should be added in future + private val REQUIRED_SUBSCRIBE_ATTRIBUTES = setOf(QueueAttribute.SUBSCRIBE.value) + private val REQUIRED_SEND_ATTRIBUTES = setOf(QueueAttribute.PUBLISH.value) } } \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt index d82c18c98..4102df682 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt @@ -29,7 +29,6 @@ import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter.Companion.DEMO_RAW_ATTRIBUTE import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Nested @@ -58,19 +57,19 @@ class TestDemoGroupBatchRouter { routingKey = "", queue = "subscribe", exchange = "test-exchange", - attributes = listOf("subscribe", DEMO_RAW_ATTRIBUTE) + attributes = listOf("subscribe") ), "test-pin1" to QueueConfiguration( routingKey = "test", queue = "", exchange = "test-exchange", - attributes = listOf("publish", DEMO_RAW_ATTRIBUTE), + attributes = listOf("publish"), ), "test-pin2" to QueueConfiguration( routingKey = "test2", queue = "", exchange = "test-exchange", - attributes = listOf("publish", "test", DEMO_RAW_ATTRIBUTE), + attributes = listOf("publish", "test"), ) ) ) @@ -105,10 +104,10 @@ class TestDemoGroupBatchRouter { @Test fun `reports about extra pins matches the publication`() { Assertions.assertThrows(IllegalStateException::class.java) { - router.send(DEMO_MESSAGE_BATCH, DEMO_RAW_ATTRIBUTE) + router.send(DEMO_MESSAGE_BATCH) }.apply { Assertions.assertEquals( - "Found incorrect number of pins [test-pin1, test-pin2] to the send operation by attributes [demo_raw, publish] and filters, expected 1, actual 2", + "Found incorrect number of pins [test-pin1, test-pin2] to the send operation by attributes [publish] and filters, expected 1, actual 2", message ) } @@ -117,10 +116,10 @@ class TestDemoGroupBatchRouter { @Test fun `reports about no pins matches the publication`() { Assertions.assertThrows(IllegalStateException::class.java) { - router.send(DEMO_MESSAGE_BATCH, DEMO_RAW_ATTRIBUTE, "unexpected") + router.send(DEMO_MESSAGE_BATCH, "unexpected") }.apply { Assertions.assertEquals( - "Found incorrect number of pins [] to the send operation by attributes [demo_raw, unexpected, publish] and filters, expected 1, actual 0", + "Found incorrect number of pins [] to the send operation by attributes [unexpected, publish] and filters, expected 1, actual 0", message ) } @@ -159,33 +158,33 @@ class TestDemoGroupBatchRouter { routingKey = "publish", queue = "", exchange = "test-exchange", - attributes = listOf("publish", DEMO_RAW_ATTRIBUTE, "test") + attributes = listOf("publish", "test") ), "test1" to QueueConfiguration( routingKey = "", queue = "queue1", exchange = "test-exchange", - attributes = listOf("subscribe", DEMO_RAW_ATTRIBUTE, "1") + attributes = listOf("subscribe", "1") ), "test2" to QueueConfiguration( routingKey = "", queue = "queue2", exchange = "test-exchange", - attributes = listOf("subscribe", DEMO_RAW_ATTRIBUTE, "2") + attributes = listOf("subscribe", "2") ) ) ) @Test fun `subscribes to correct queue`() { - val monitor = router.subscribe(mock { }, "1", DEMO_RAW_ATTRIBUTE) + val monitor = router.subscribe(mock { }, "1") Assertions.assertNotNull(monitor) { "monitor must not be null" } verify(connectionManager).basicConsume(eq("queue1"), any(), any()) } @Test fun `subscribes with manual ack to correct queue`() { - val monitor = router.subscribeWithManualAck(mock { }, "1", DEMO_RAW_ATTRIBUTE) + val monitor = router.subscribeWithManualAck(mock { }, "1") Assertions.assertNotNull(monitor) { "monitor must not be null" } verify(connectionManager).basicConsume(eq("queue1"), any(), any()) @@ -193,7 +192,7 @@ class TestDemoGroupBatchRouter { @Test fun `subscribes to all matched queues`() { - val monitor = router.subscribeAll(mock { }, DEMO_RAW_ATTRIBUTE) + val monitor = router.subscribeAll(mock { }) Assertions.assertNotNull(monitor) { "monitor must not be null" } verify(connectionManager).basicConsume(eq("queue1"), any(), any()) @@ -205,7 +204,7 @@ class TestDemoGroupBatchRouter { Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }) } .apply { Assertions.assertEquals( - "Found incorrect number of pins [test1, test2] to subscribe operation by attributes [demo_raw, subscribe] and filters, expected 1, actual 2", + "Found incorrect number of pins [test1, test2] to subscribe operation by attributes [subscribe] and filters, expected 1, actual 2", message ) } @@ -215,19 +214,19 @@ class TestDemoGroupBatchRouter { fun `reports if no queue matches`() { Assertions.assertAll( Executable { - Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }, DEMO_RAW_ATTRIBUTE, "unexpected") } + Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }, "unexpected") } .apply { Assertions.assertEquals( - "Found incorrect number of pins [] to subscribe operation by attributes [demo_raw, unexpected, subscribe] and filters, expected 1, actual 0", + "Found incorrect number of pins [] to subscribe operation by attributes [unexpected, subscribe] and filters, expected 1, actual 0", message ) } }, Executable { - Assertions.assertThrows(IllegalStateException::class.java) { router.subscribeAll(mock { }, DEMO_RAW_ATTRIBUTE, "unexpected") } + Assertions.assertThrows(IllegalStateException::class.java) { router.subscribeAll(mock { }, "unexpected") } .apply { Assertions.assertEquals( - "Found incorrect number of pins [] to subscribe all operation by attributes [demo_raw, unexpected, subscribe] and filters, expected 1 or more, actual 0", + "Found incorrect number of pins [] to subscribe all operation by attributes [unexpected, subscribe] and filters, expected 1 or more, actual 0", message ) } From 03a82588aea20f1df65d351a910f8d1b8b500ba7 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 28 Mar 2023 16:41:45 +0400 Subject: [PATCH 110/154] Write body content when message can not be decoded --- .../message/impl/rabbitmq/AbstractRabbitSubscriber.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index 38bb57099..3547e1853 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -25,6 +25,7 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import com.google.common.base.Suppliers; +import com.google.common.io.BaseEncoding; import com.rabbitmq.client.Delivery; import io.prometheus.client.Counter; import io.prometheus.client.Histogram; @@ -267,7 +268,9 @@ private void handle(DeliveryMetadata deliveryMetadata, try { value = valueFromBytes(delivery.getBody()); } catch (Exception e) { - LOGGER.error("Couldn't parse delivery. Reject message received", e); + if (LOGGER.isErrorEnabled()) { + LOGGER.error("Couldn't parse delivery: {}. Reject message received", BaseEncoding.base16().encode(delivery.getBody()), e); + } confirmProcessed.reject(); throw new IOException( String.format( From 7bcec302219132c9b81071fdcfd5d84c8ab86b1b Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Tue, 28 Mar 2023 18:01:57 +0400 Subject: [PATCH 111/154] Add parsed messages to interop protocol --- build.gradle | 1 + .../schema/message/impl/rabbitmq/demo/Demo.kt | 60 ++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 53c0a6644..2de827a5c 100644 --- a/build.gradle +++ b/build.gradle @@ -217,6 +217,7 @@ dependencies { } implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" implementation "com.fasterxml.jackson.module:jackson-module-kotlin" + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor' implementation 'com.fasterxml.uuid:java-uuid-generator:4.0.1' diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt index b3c8495d8..734735864 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt @@ -18,9 +18,17 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoDirection.INCOMING import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoDirection.OUTGOING +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.dataformat.cbor.databind.CBORMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.jacksonTypeRef import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufInputStream +import io.netty.buffer.ByteBufOutputStream import io.netty.buffer.ByteBufUtil import io.netty.buffer.Unpooled +import java.io.InputStream +import java.io.OutputStream import java.nio.charset.Charset import java.time.Instant import java.util.Objects @@ -43,6 +51,7 @@ enum class ValueType(val codec: ValueCodec<*>) { PROTOCOL(ProtocolCodec), RAW_MESSAGE(RawMessageCodec), RAW_MESSAGE_BODY(RawMessageBodyCodec), + PARSED_MESSAGE(ParsedMessageCodec), MESSAGE_GROUP(MessageGroupCodec), MESSAGE_LIST(MessageListCodec), GROUP_BATCH(GroupBatchCodec), @@ -154,6 +163,20 @@ abstract class InstantCodec(type: UByte) : AbstractCodec(type) { } } +open class CborCodec(type: UByte, private val typeReference: TypeReference) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): T = ByteBufInputStream(buffer).use { + return MAPPER.readValue(it as InputStream, typeReference) + } + + override fun write(buffer: ByteBuf, value: T) = ByteBufOutputStream(buffer).use { + MAPPER.writeValue(it as OutputStream, value) + } + + companion object { + private val MAPPER = CBORMapper().registerModule(JavaTimeModule()) + } +} + object LongTypeCodec : LongCodec(1u) object StringTypeCodec : StringCodec(2u) @@ -228,6 +251,8 @@ object RawMessageCodec : AbstractCodec(20u) { object RawMessageBodyCodec : ByteArrayCodec(21u) +object ParsedMessageCodec : CborCodec(30u, jacksonTypeRef()) + object MessageGroupCodec : AbstractCodec(40u) { override fun read(buffer: ByteBuf): DemoMessageGroup = DemoMessageGroup().apply { buffer.forEachValue { codec -> @@ -248,6 +273,7 @@ object MessageListCodec : AbstractCodec>>(41u) { buffer.forEachValue { codec -> when (codec) { is RawMessageCodec -> this += codec.decode(buffer) + is ParsedMessageCodec -> this += codec.decode(buffer) else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") } } @@ -256,6 +282,7 @@ object MessageListCodec : AbstractCodec>>(41u) { override fun write(buffer: ByteBuf, value: List>): Unit = value.forEach { message -> when (message) { is DemoRawMessage -> RawMessageCodec.encode(message, buffer) + is DemoParsedMessage -> ParsedMessageCodec.encode(message, buffer) else -> println("Skipping unsupported message type: $message") } } @@ -357,6 +384,14 @@ data class DemoRawMessage( } } +data class DemoParsedMessage( + override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, + override var metadata: Map = mapOf(), + override var protocol: String = "", + var type: String = "", + override var body: Map = mapOf(), +) : DemoMessage> + data class DemoMessageGroup( var messages: List> = listOf(), ) @@ -411,10 +446,33 @@ fun main() { body = byteArrayOf(5, 6, 7, 8) ) + val message3 = DemoParsedMessage( + id = DemoMessageId( + book = "book1", + sessionGroup = "group1", + sessionAlias = "alias3", + direction = OUTGOING, + sequence = 3, + subsequence = listOf(5, 6), + timestamp = Instant.now() + ), + metadata = mapOf( + "prop5" to "value6", + "prop7" to "value8" + ), + protocol = "proto3", + type = "some-type", + body = mapOf( + "simple" to 1, + "list" to listOf(1, 2, 3), + "map" to mapOf("abc" to "cde") + ) + ) + val batch = DemoGroupBatch( book = "book1", sessionGroup = "group1", - groups = listOf(DemoMessageGroup(listOf(message1, message2))) + groups = listOf(DemoMessageGroup(listOf(message1, message2, message3))) ) GroupBatchCodec.encode(batch, buffer) From e1ef64b567a81383420f497f35da23821e5a0eec Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Tue, 28 Mar 2023 18:14:37 +0400 Subject: [PATCH 112/154] Change parsed message encoding --- .../schema/message/impl/rabbitmq/demo/Demo.kt | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt index 734735864..9eacd2720 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt @@ -49,9 +49,11 @@ enum class ValueType(val codec: ValueCodec<*>) { TIMESTAMP(TimestampCodec), METADATA(MetadataCodec), PROTOCOL(ProtocolCodec), + MESSAGE_TYPE(MessageTypeCodec), RAW_MESSAGE(RawMessageCodec), RAW_MESSAGE_BODY(RawMessageBodyCodec), PARSED_MESSAGE(ParsedMessageCodec), + PARSED_MESSAGE_BODY(ParsedMessageBodyCodec), MESSAGE_GROUP(MessageGroupCodec), MESSAGE_LIST(MessageListCodec), GROUP_BATCH(GroupBatchCodec), @@ -228,6 +230,8 @@ object MetadataCodec : MapCodec(11u, StringTypeCodec, StringType object ProtocolCodec : StringCodec(12u) +object MessageTypeCodec : StringCodec(13u) + object RawMessageCodec : AbstractCodec(20u) { override fun read(buffer: ByteBuf): DemoRawMessage = DemoRawMessage().apply { buffer.forEachValue { codec -> @@ -251,7 +255,30 @@ object RawMessageCodec : AbstractCodec(20u) { object RawMessageBodyCodec : ByteArrayCodec(21u) -object ParsedMessageCodec : CborCodec(30u, jacksonTypeRef()) +object ParsedMessageCodec : AbstractCodec(30u) { + override fun read(buffer: ByteBuf): DemoParsedMessage = DemoParsedMessage().apply { + buffer.forEachValue { codec -> + when (codec) { + is MessageIdCodec -> id = codec.decode(buffer) + is MetadataCodec -> metadata = codec.decode(buffer) + is ProtocolCodec -> protocol = codec.decode(buffer) + is MessageTypeCodec -> type = codec.decode(buffer) + is ParsedMessageBodyCodec -> body = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: DemoParsedMessage) { + MessageIdCodec.encode(value.id, buffer) + MetadataCodec.encode(value.metadata, buffer) + ProtocolCodec.encode(value.protocol, buffer) + MessageTypeCodec.encode(value.type, buffer) + ParsedMessageBodyCodec.encode(value.body, buffer) + } +} + +object ParsedMessageBodyCodec : CborCodec>(31u, jacksonTypeRef()) object MessageGroupCodec : AbstractCodec(40u) { override fun read(buffer: ByteBuf): DemoMessageGroup = DemoMessageGroup().apply { From f5c2f7f2f04539cee13f4678ed943550dfe8f302 Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Wed, 29 Mar 2023 12:11:14 +0400 Subject: [PATCH 113/154] Interproto > Decoding > Make 'ByteBuf.forEachValue' inline to avoid lambda allocations --- .../th2/common/schema/message/impl/rabbitmq/demo/Demo.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt index 9eacd2720..76aaa4d4a 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt @@ -344,7 +344,7 @@ object GroupBatchCodec : AbstractCodec(50u) { object GroupListCodec : ListCodec(51u, MessageGroupCodec) -fun ByteBuf.forEachValue(action: (codec: ValueCodec<*>) -> Unit) { +inline fun ByteBuf.forEachValue(action: (codec: ValueCodec<*>) -> Unit) { while (isReadable) { val type = getByte(readerIndex()).toUByte() From 258da89645e51b34b8fbc1772163e36fd2aed9a5 Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Wed, 29 Mar 2023 15:18:50 +0400 Subject: [PATCH 114/154] Interproto > Change raw message body type to ByteBuf --- .../schema/message/impl/rabbitmq/demo/Demo.kt | 40 ++++++------------- .../rabbitmq/demo/TestDemoGroupBatchRouter.kt | 5 ++- 2 files changed, 15 insertions(+), 30 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt index 76aaa4d4a..22f5f4283 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt @@ -31,7 +31,6 @@ import java.io.InputStream import java.io.OutputStream import java.nio.charset.Charset import java.time.Instant -import java.util.Objects // TODO: maybe make length field a variable length int @@ -149,11 +148,11 @@ abstract class MapCodec( } } -abstract class ByteArrayCodec(type: UByte) : AbstractCodec(type) { - override fun read(buffer: ByteBuf): ByteArray = ByteArray(buffer.readableBytes()).apply(buffer::readBytes) +abstract class ByteBufCodec(type: UByte) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): ByteBuf = buffer.copy() - override fun write(buffer: ByteBuf, value: ByteArray) { - buffer.writeBytes(value) + override fun write(buffer: ByteBuf, value: ByteBuf) { + value.markReaderIndex().apply(buffer::writeBytes).resetReaderIndex() } } @@ -253,7 +252,7 @@ object RawMessageCodec : AbstractCodec(20u) { } } -object RawMessageBodyCodec : ByteArrayCodec(21u) +object RawMessageBodyCodec : ByteBufCodec(21u) object ParsedMessageCodec : AbstractCodec(30u) { override fun read(buffer: ByteBuf): DemoParsedMessage = DemoParsedMessage().apply { @@ -326,9 +325,9 @@ object GroupBatchCodec : AbstractCodec(50u) { } } - groups.forEach { - it.messages.forEach { - val id = it.id + groups.forEach { group -> + group.messages.forEach { message -> + val id = message.id id.book = book id.sessionGroup = sessionGroup } @@ -393,23 +392,8 @@ data class DemoRawMessage( override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, override var metadata: Map = mapOf(), override var protocol: String = "", - override var body: ByteArray = EMPTY_BODY, -) : DemoMessage { - override fun equals(other: Any?): Boolean = when { - this === other -> true - other !is DemoRawMessage -> false - id != other.id -> false - metadata != other.metadata -> false - protocol != other.protocol -> false - else -> body.contentEquals(other.body) - } - - override fun hashCode(): Int = Objects.hash(id, metadata, protocol, body.contentHashCode()) - - companion object { - private val EMPTY_BODY = ByteArray(0) - } -} + override var body: ByteBuf = Unpooled.EMPTY_BUFFER, +) : DemoMessage data class DemoParsedMessage( override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, @@ -452,7 +436,7 @@ fun main() { "prop2" to "value2" ), protocol = "proto1", - body = byteArrayOf(1, 2, 3, 4) + body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3, 4)) ) val message2 = DemoRawMessage( @@ -470,7 +454,7 @@ fun main() { "prop4" to "value4" ), protocol = "proto2", - body = byteArrayOf(5, 6, 7, 8) + body = Unpooled.wrappedBuffer(byteArrayOf(5, 6, 7, 8)) ) val message3 = DemoParsedMessage( diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt index 4102df682..ac104b54c 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt @@ -29,6 +29,7 @@ import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import io.netty.buffer.Unpooled import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Nested @@ -84,7 +85,7 @@ class TestDemoGroupBatchRouter { listOf( DemoRawMessage( DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), - body = byteArrayOf(1, 2, 3) + body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3)) ) ) ) @@ -256,7 +257,7 @@ class TestDemoGroupBatchRouter { listOf( DemoRawMessage( DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), - body = byteArrayOf(1, 2, 3) + body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3)) ) ) ) From 6986b77bdcf080bdb0886a414615b08604fca513 Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Wed, 29 Mar 2023 16:55:36 +0400 Subject: [PATCH 115/154] Interproto > Add event ID to messages, make lists mutable --- .../schema/message/impl/rabbitmq/demo/Demo.kt | 68 +++++++++++++++---- .../rabbitmq/demo/TestDemoGroupBatchRouter.kt | 8 +-- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt index 22f5f4283..e7b864ad3 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt @@ -31,6 +31,7 @@ import java.io.InputStream import java.io.OutputStream import java.nio.charset.Charset import java.time.Instant +import java.util.Collections.emptyList // TODO: maybe make length field a variable length int @@ -49,6 +50,9 @@ enum class ValueType(val codec: ValueCodec<*>) { METADATA(MetadataCodec), PROTOCOL(ProtocolCodec), MESSAGE_TYPE(MessageTypeCodec), + ID_CODEC(IdCodec), + SCOPE_CODEC(ScopeCodec), + EVENT_ID_CODEC(EventIdCodec), RAW_MESSAGE(RawMessageCodec), RAW_MESSAGE_BODY(RawMessageBodyCodec), PARSED_MESSAGE(ParsedMessageCodec), @@ -119,14 +123,14 @@ abstract class LongCodec(type: UByte) : AbstractCodec(type) { } } -abstract class ListCodec(type: UByte, private val elementCodec: ValueCodec) : AbstractCodec>(type) { - override fun read(buffer: ByteBuf): List = mutableListOf().also { list -> +abstract class ListCodec(type: UByte, private val elementCodec: ValueCodec) : AbstractCodec>(type) { + override fun read(buffer: ByteBuf): MutableList = mutableListOf().also { list -> while (buffer.isReadable) { list += elementCodec.decode(buffer) } } - override fun write(buffer: ByteBuf, value: List) { + override fun write(buffer: ByteBuf, value: MutableList) { value.forEach { elementCodec.encode(it, buffer) } } } @@ -231,11 +235,36 @@ object ProtocolCodec : StringCodec(12u) object MessageTypeCodec : StringCodec(13u) +object IdCodec : StringCodec(14u) + +object ScopeCodec : StringCodec(15u) + +object EventIdCodec : AbstractCodec(16u) { + override fun read(buffer: ByteBuf): DemoEventId = DemoEventId().apply { + buffer.forEachValue { codec -> + when (codec) { + is IdCodec -> id = codec.decode(buffer) + is BookCodec -> book = codec.decode(buffer) + is ScopeCodec -> scope = codec.decode(buffer) + is TimestampCodec -> timestamp = codec.decode(buffer) + } + } + } + + override fun write(buffer: ByteBuf, value: DemoEventId) { + IdCodec.encode(value.id, buffer) + BookCodec.encode(value.book, buffer) + ScopeCodec.encode(value.scope, buffer) + TimestampCodec.encode(value.timestamp, buffer) + } +} + object RawMessageCodec : AbstractCodec(20u) { override fun read(buffer: ByteBuf): DemoRawMessage = DemoRawMessage().apply { buffer.forEachValue { codec -> when (codec) { is MessageIdCodec -> id = codec.decode(buffer) + is EventIdCodec -> eventId = codec.decode(buffer) is MetadataCodec -> metadata = codec.decode(buffer) is ProtocolCodec -> protocol = codec.decode(buffer) is RawMessageBodyCodec -> body = codec.decode(buffer) @@ -246,6 +275,7 @@ object RawMessageCodec : AbstractCodec(20u) { override fun write(buffer: ByteBuf, value: DemoRawMessage) { MessageIdCodec.encode(value.id, buffer) + value.eventId?.run { EventIdCodec.encode(this, buffer) } MetadataCodec.encode(value.metadata, buffer) ProtocolCodec.encode(value.protocol, buffer) RawMessageBodyCodec.encode(value.body, buffer) @@ -259,6 +289,7 @@ object ParsedMessageCodec : AbstractCodec(30u) { buffer.forEachValue { codec -> when (codec) { is MessageIdCodec -> id = codec.decode(buffer) + is EventIdCodec -> eventId = codec.decode(buffer) is MetadataCodec -> metadata = codec.decode(buffer) is ProtocolCodec -> protocol = codec.decode(buffer) is MessageTypeCodec -> type = codec.decode(buffer) @@ -270,6 +301,7 @@ object ParsedMessageCodec : AbstractCodec(30u) { override fun write(buffer: ByteBuf, value: DemoParsedMessage) { MessageIdCodec.encode(value.id, buffer) + value.eventId?.run { EventIdCodec.encode(this, buffer) } MetadataCodec.encode(value.metadata, buffer) ProtocolCodec.encode(value.protocol, buffer) MessageTypeCodec.encode(value.type, buffer) @@ -294,8 +326,8 @@ object MessageGroupCodec : AbstractCodec(40u) { } } -object MessageListCodec : AbstractCodec>>(41u) { - override fun read(buffer: ByteBuf): List> = mutableListOf>().apply { +object MessageListCodec : AbstractCodec>>(41u) { + override fun read(buffer: ByteBuf): MutableList> = mutableListOf>().apply { buffer.forEachValue { codec -> when (codec) { is RawMessageCodec -> this += codec.decode(buffer) @@ -305,7 +337,7 @@ object MessageListCodec : AbstractCodec>>(41u) { } } - override fun write(buffer: ByteBuf, value: List>): Unit = value.forEach { message -> + override fun write(buffer: ByteBuf, value: MutableList>): Unit = value.forEach { message -> when (message) { is DemoRawMessage -> RawMessageCodec.encode(message, buffer) is DemoParsedMessage -> ParsedMessageCodec.encode(message, buffer) @@ -373,7 +405,7 @@ data class DemoMessageId( var sessionAlias: String = "", var direction: DemoDirection = INCOMING, var sequence: Long = 0, - var subsequence: List = listOf(), + var subsequence: MutableList = emptyList(), var timestamp: Instant = Instant.EPOCH, ) { companion object { @@ -381,8 +413,16 @@ data class DemoMessageId( } } +data class DemoEventId( + var id: String = "", + var book: String = "", + var scope: String = "", + var timestamp: Instant = Instant.EPOCH, +) + interface DemoMessage { var id: DemoMessageId + var eventId: DemoEventId? var metadata: Map var protocol: String var body: T @@ -390,6 +430,7 @@ interface DemoMessage { data class DemoRawMessage( override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, + override var eventId: DemoEventId? = null, override var metadata: Map = mapOf(), override var protocol: String = "", override var body: ByteBuf = Unpooled.EMPTY_BUFFER, @@ -397,6 +438,7 @@ data class DemoRawMessage( data class DemoParsedMessage( override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, + override var eventId: DemoEventId? = null, override var metadata: Map = mapOf(), override var protocol: String = "", var type: String = "", @@ -404,13 +446,13 @@ data class DemoParsedMessage( ) : DemoMessage> data class DemoMessageGroup( - var messages: List> = listOf(), + var messages: MutableList> = emptyList(), // FIXME: default value isn't really mutable ) data class DemoGroupBatch( var book: String = "", var sessionGroup: String = "", - var groups: List = listOf(), + var groups: MutableList = emptyList(), ) fun DemoGroupBatch.toByteArray() = Unpooled.buffer().run { @@ -428,7 +470,7 @@ fun main() { sessionAlias = "alias1", direction = INCOMING, sequence = 1, - subsequence = listOf(1, 2), + subsequence = mutableListOf(1, 2), timestamp = Instant.now() ), metadata = mapOf( @@ -446,7 +488,7 @@ fun main() { sessionAlias = "alias2", direction = OUTGOING, sequence = 2, - subsequence = listOf(3, 4), + subsequence = mutableListOf(3, 4), timestamp = Instant.now() ), metadata = mapOf( @@ -464,7 +506,7 @@ fun main() { sessionAlias = "alias3", direction = OUTGOING, sequence = 3, - subsequence = listOf(5, 6), + subsequence = mutableListOf(5, 6), timestamp = Instant.now() ), metadata = mapOf( @@ -483,7 +525,7 @@ fun main() { val batch = DemoGroupBatch( book = "book1", sessionGroup = "group1", - groups = listOf(DemoMessageGroup(listOf(message1, message2, message3))) + groups = mutableListOf(DemoMessageGroup(mutableListOf(message1, message2, message3))) ) GroupBatchCodec.encode(batch, buffer) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt index ac104b54c..82f922f53 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt @@ -80,9 +80,9 @@ class TestDemoGroupBatchRouter { val batch = DemoGroupBatch( BOOK_NAME, SESSION_GROUP, - listOf( + mutableListOf( DemoMessageGroup( - listOf( + mutableListOf( DemoRawMessage( DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3)) @@ -252,9 +252,9 @@ class TestDemoGroupBatchRouter { private val DEMO_MESSAGE_BATCH = DemoGroupBatch( BOOK_NAME, SESSION_GROUP, - listOf( + mutableListOf( DemoMessageGroup( - listOf( + mutableListOf( DemoRawMessage( DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3)) From a4a45720c4abea9e9d45861af9d4339e76db7201 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 29 Mar 2023 18:14:11 +0400 Subject: [PATCH 116/154] Added Cleanable interface for demo protocol --- .../schema/message/impl/rabbitmq/demo/Demo.kt | 242 ++++++++++++++++-- 1 file changed, 220 insertions(+), 22 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt index e7b864ad3..44267bc42 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt @@ -18,6 +18,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoDirection.INCOMING import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoDirection.OUTGOING +import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessage.Companion.DEFAULT_METADATA import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.dataformat.cbor.databind.CBORMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule @@ -31,6 +32,7 @@ import java.io.InputStream import java.io.OutputStream import java.nio.charset.Charset import java.time.Instant +import java.util.* import java.util.Collections.emptyList // TODO: maybe make length field a variable length int @@ -139,14 +141,14 @@ abstract class MapCodec( type: UByte, private val keyCodec: ValueCodec, private val valueCodec: ValueCodec, -) : AbstractCodec>(type) { - override fun read(buffer: ByteBuf): Map = mutableMapOf().apply { +) : AbstractCodec>(type) { + override fun read(buffer: ByteBuf): MutableMap = mutableMapOf().apply { while (buffer.isReadable) { this[keyCodec.decode(buffer)] = valueCodec.decode(buffer) } } - override fun write(buffer: ByteBuf, value: Map): Unit = value.forEach { (key, value) -> + override fun write(buffer: ByteBuf, value: MutableMap): Unit = value.forEach { (key, value) -> keyCodec.encode(key, buffer) valueCodec.encode(value, buffer) } @@ -309,7 +311,7 @@ object ParsedMessageCodec : AbstractCodec(30u) { } } -object ParsedMessageBodyCodec : CborCodec>(31u, jacksonTypeRef()) +object ParsedMessageBodyCodec : CborCodec>(31u, jacksonTypeRef()) object MessageGroupCodec : AbstractCodec(40u) { override fun read(buffer: ByteBuf): DemoMessageGroup = DemoMessageGroup().apply { @@ -399,17 +401,52 @@ enum class DemoDirection(val id: Int) { } } +interface Cleanable { + /** + * Cleans all embedded th2 objects + */ + fun clean() + + /** + * Cleans only current layer include collections. + */ + fun softClean() = clean() +} + data class DemoMessageId( var book: String = "", var sessionGroup: String = "", var sessionAlias: String = "", var direction: DemoDirection = INCOMING, var sequence: Long = 0, - var subsequence: MutableList = emptyList(), + /** The subsequence is not mutable by default */ + var subsequence: MutableList = DEFAULT_SUBSEQUENCE, var timestamp: Instant = Instant.EPOCH, -) { +) : Cleanable { + + override fun clean() { + check(this !== DEFAULT_INSTANCE) { + "Object can be cleaned because it is default instance" + } + check(subsequence !== DEFAULT_SUBSEQUENCE) { + "Object can be cleaned because 'subsequence' is immutable" + } + + book = "" + sessionGroup = "" + sessionAlias = "" + direction = INCOMING + sequence = 0 + subsequence.clear() + timestamp = Instant.EPOCH + } + companion object { + val DEFAULT_SUBSEQUENCE: MutableList = emptyList() val DEFAULT_INSTANCE: DemoMessageId = DemoMessageId() // FIXME: do smth about its mutability + fun newMutable() = DemoMessageId( + subsequence = mutableListOf() + ) } } @@ -418,42 +455,184 @@ data class DemoEventId( var book: String = "", var scope: String = "", var timestamp: Instant = Instant.EPOCH, -) +) : Cleanable { -interface DemoMessage { + override fun clean() { + check(this !== DEFAULT_INSTANCE) { + "Object can be cleaned because it is default instance" + } + id= "" + book = "" + scope = "" + timestamp = Instant.EPOCH + } + + companion object { + val DEFAULT_INSTANCE: DemoEventId = DemoEventId() // FIXME: do smth about its mutability + fun newMutable() = DemoEventId() + } +} + +interface DemoMessage : Cleanable { + /** The id is not mutable by default */ var id: DemoMessageId var eventId: DemoEventId? - var metadata: Map + /** The metadata is not mutable by default */ + var metadata: MutableMap var protocol: String var body: T + + companion object { + val DEFAULT_METADATA: MutableMap = Collections.emptyMap() + } } data class DemoRawMessage( override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, override var eventId: DemoEventId? = null, - override var metadata: Map = mapOf(), + override var metadata: MutableMap = DEFAULT_METADATA, override var protocol: String = "", + /** The body is not mutable by default */ override var body: ByteBuf = Unpooled.EMPTY_BUFFER, -) : DemoMessage +) : DemoMessage { + override fun clean() { + check(id !== DemoMessageId.DEFAULT_INSTANCE) { + "Object can be cleaned because 'id' is default instance" + } + check(metadata !== DEFAULT_METADATA) { + "Object can be cleaned because 'metadata' is immutable" + } + check(body !== Unpooled.EMPTY_BUFFER) { + "Object can be cleaned because 'body' is immutable" + } + + id.clean() + eventId = null + metadata.clear() + protocol = "" + body.clear() + } + + override fun softClean() { + check(metadata !== DEFAULT_METADATA) { + "Object can be cleaned because 'metadata' is immutable" + } + + id = DemoMessageId.DEFAULT_INSTANCE + eventId = null + metadata.clear() + protocol = "" + body = Unpooled.EMPTY_BUFFER + } + + companion object { + fun newMutable() = DemoRawMessage( + id = DemoMessageId.newMutable(), + metadata = mutableMapOf(), + body = Unpooled.buffer() + ) + fun newSoftMutable() = DemoRawMessage( + metadata = mutableMapOf() + ) + } +} data class DemoParsedMessage( override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, override var eventId: DemoEventId? = null, - override var metadata: Map = mapOf(), + override var metadata: MutableMap = DEFAULT_METADATA, override var protocol: String = "", var type: String = "", - override var body: Map = mapOf(), -) : DemoMessage> + /** The body is not mutable by default */ + override var body: MutableMap = DEFAULT_BODY, +) : DemoMessage> { + override fun clean() { + check(id !== DemoMessageId.DEFAULT_INSTANCE) { + "Object can be cleaned because 'id' is default instance" + } + check(metadata !== DEFAULT_METADATA) { + "Object can be cleaned because 'metadata' is immutable" + } + check(body !== DEFAULT_BODY) { + "Object can be cleaned because 'body' is immutable" + } + + id.clean() + eventId = null + metadata.clear() + protocol = "" + type = "" + body.clear() + } + + override fun softClean() { + check(metadata !== DEFAULT_METADATA) { + "Object can be cleaned because 'metadata' is immutable" + } + check(body !== DEFAULT_BODY) { + "Object can be cleaned because 'body' is immutable" + } + + id = DemoMessageId.DEFAULT_INSTANCE + eventId = null + metadata.clear() + protocol = "" + type = "" + body.clear() + } + + companion object { + val DEFAULT_BODY: MutableMap = Collections.emptyMap() + fun newMutable() = DemoParsedMessage( + id = DemoMessageId.newMutable(), + metadata = mutableMapOf(), + body = mutableMapOf() + ) + fun newSoftMutable() = DemoParsedMessage( + metadata = mutableMapOf(), + body = mutableMapOf() + ) + } +} data class DemoMessageGroup( - var messages: MutableList> = emptyList(), // FIXME: default value isn't really mutable -) + var messages: MutableList> = DEFAULT_MESSAGES, +) : Cleanable { + override fun clean() { + check(messages !== DEFAULT_MESSAGES) { + "Object can be cleaned because 'messages' is immutable" + } + messages.clear() + } + + companion object { + val DEFAULT_MESSAGES: MutableList> = emptyList() + fun newMutable() = DemoMessageGroup(mutableListOf()) + } +} data class DemoGroupBatch( var book: String = "", var sessionGroup: String = "", - var groups: MutableList = emptyList(), -) + var groups: MutableList = DEFAULT_GROUPS, +) : Cleanable { + override fun clean() { + check(groups !== DEFAULT_GROUPS) { + "Object can be cleaned because 'groups' is immutable" + } + + book = "" + sessionGroup = "" + groups.clear() + } + + companion object { + val DEFAULT_GROUPS: MutableList = emptyList() + fun newMutable() = DemoGroupBatch( + groups = mutableListOf() + ) + } +} fun DemoGroupBatch.toByteArray() = Unpooled.buffer().run { GroupBatchCodec.encode(this@toByteArray, this@run) @@ -461,6 +640,25 @@ fun DemoGroupBatch.toByteArray() = Unpooled.buffer().run { } fun main() { + DemoMessageId.newMutable().apply(DemoMessageId::clean) + .apply(DemoMessageId::softClean) + DemoEventId.newMutable().apply(DemoEventId::clean) + .apply(DemoEventId::softClean) + + DemoRawMessage.newMutable().apply(DemoRawMessage::clean) + .apply(DemoRawMessage::softClean) + DemoRawMessage.newSoftMutable().apply(DemoRawMessage::softClean) + + DemoParsedMessage.newMutable().apply(DemoParsedMessage::clean) + .apply(DemoParsedMessage::softClean) + DemoParsedMessage.newSoftMutable().apply(DemoParsedMessage::softClean) + + DemoMessageGroup.newMutable().apply(DemoMessageGroup::clean) + .apply(DemoMessageGroup::softClean) + + DemoGroupBatch.newMutable().apply(DemoGroupBatch::clean) + .apply(DemoGroupBatch::softClean) + val buffer = Unpooled.buffer() val message1 = DemoRawMessage( @@ -473,7 +671,7 @@ fun main() { subsequence = mutableListOf(1, 2), timestamp = Instant.now() ), - metadata = mapOf( + metadata = mutableMapOf( "prop1" to "value1", "prop2" to "value2" ), @@ -491,7 +689,7 @@ fun main() { subsequence = mutableListOf(3, 4), timestamp = Instant.now() ), - metadata = mapOf( + metadata = mutableMapOf( "prop3" to "value3", "prop4" to "value4" ), @@ -509,13 +707,13 @@ fun main() { subsequence = mutableListOf(5, 6), timestamp = Instant.now() ), - metadata = mapOf( + metadata = mutableMapOf( "prop5" to "value6", "prop7" to "value8" ), protocol = "proto3", type = "some-type", - body = mapOf( + body = mutableMapOf( "simple" to 1, "list" to listOf(1, 2, 3), "map" to mapOf("abc" to "cde") From 75f3a4c8db99c100df10f6db9a824f31b889771c Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Thu, 13 Apr 2023 17:28:24 +0400 Subject: [PATCH 117/154] Refactored name of transport protocol components --- build.gradle | 16 +- .../schema/factory/AbstractCommonFactory.java | 15 +- .../impl/AbstractTh2MsgFilterStrategy.java | 8 +- .../th2/common/metrics/MetricsUtils.kt | 2 - .../schema/cradle/CradleConfiguration.kt | 4 +- .../strategy/impl/AnyMessageFilterStrategy.kt | 18 +- .../impl/rabbitmq/AbstractRabbitRouter.kt | 26 +- .../schema/message/impl/rabbitmq/demo/Demo.kt | 737 ------------------ .../group/RabbitMessageGroupBatchRouter.kt | 18 +- .../RabbitMessageGroupBatchSubscriber.kt | 2 +- .../impl/rabbitmq/transport/Cleanable.kt | 30 + .../message/impl/rabbitmq/transport/Codecs.kt | 407 ++++++++++ .../impl/rabbitmq/transport/Direction.kt | 36 + .../impl/rabbitmq/transport/EventId.kt | 42 + .../impl/rabbitmq/transport/GroupBatch.kt | 40 + .../impl/rabbitmq/transport/Message.kt | 33 + .../impl/rabbitmq/transport/MessageGroup.kt | 35 + .../impl/rabbitmq/transport/MessageId.kt | 56 ++ .../impl/rabbitmq/transport/ParsedMessage.kt | 77 ++ .../impl/rabbitmq/transport/RawMessage.kt | 70 ++ .../TransportGroupBatchRouter.kt} | 39 +- .../TransportGroupBatchSender.kt} | 26 +- .../TransportGroupBatchSubscriber.kt} | 31 +- .../impl/rabbitmq/transport/TransportUtils.kt | 74 ++ .../th2/common/event/bean/BaseTest.java | 2 +- .../impl/rabbitmq/transport/CleanableTest.kt | 173 ++++ .../impl/rabbitmq/transport/CodecsTest.kt | 100 +++ ...ansportGroupBatchRouterIntegrationTest.kt} | 10 +- .../TransportGroupBatchRouterTest.kt} | 166 ++-- .../rabbitmq/transport/TransportUtilsTest.kt | 116 +++ 30 files changed, 1496 insertions(+), 913 deletions(-) delete mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Cleanable.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Direction.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt rename src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/{demo/DemoMessageBatchRouter.kt => transport/TransportGroupBatchRouter.kt} (65%) rename src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/{demo/DemoMessageBatchSender.kt => transport/TransportGroupBatchSender.kt} (65%) rename src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/{demo/DemoMessageBatchSubscriber.kt => transport/TransportGroupBatchSubscriber.kt} (62%) create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt rename src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/{demo/IntegrationTestDemoGroupBatchRouter.kt => transport/TransportGroupBatchRouterIntegrationTest.kt} (96%) rename src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/{demo/TestDemoGroupBatchRouter.kt => transport/TransportGroupBatchRouterTest.kt} (62%) create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt diff --git a/build.gradle b/build.gradle index 2de827a5c..5bd3a266d 100644 --- a/build.gradle +++ b/build.gradle @@ -177,9 +177,7 @@ tasks.register('integrationTest', Test) { dependencies { api platform("com.exactpro.th2:bom:4.2.0") api 'com.exactpro.th2:grpc-common:4.2.0-dev' - api ("com.exactpro.th2:cradle-core:${cradleVersion}") { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability - } + api "com.exactpro.th2:cradle-core:${cradleVersion}" api 'io.github.microutils:kotlin-logging:2.1.21' jmh 'org.openjdk.jmh:jmh-core:0.9' @@ -187,10 +185,7 @@ dependencies { implementation 'com.google.protobuf:protobuf-java-util' implementation 'com.exactpro.th2:grpc-service-generator:3.2.2' - api ("com.exactpro.th2:cradle-cassandra:${cradleVersion}") { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' // because of the vulnerability - } - implementation "com.datastax.oss:java-driver-core" + implementation "com.exactpro.th2:cradle-cassandra:${cradleVersion}" //FIXME: Add these dependencies as api to grpc-... artifacts implementation "io.grpc:grpc-protobuf" @@ -203,16 +198,16 @@ dependencies { implementation "org.jetbrains:annotations" implementation "org.apache.commons:commons-lang3" - implementation "commons-io:commons-io" implementation "org.apache.commons:commons-collections4" implementation "org.apache.commons:commons-text" implementation "commons-cli:commons-cli" + implementation "commons-io:commons-io" implementation "com.fasterxml.jackson.core:jackson-core" - api("com.fasterxml.jackson.core:jackson-databind") { + implementation("com.fasterxml.jackson.core:jackson-databind") { because('provide ability to use object mapper in components') } - api("com.fasterxml.jackson.core:jackson-annotations") { + implementation("com.fasterxml.jackson.core:jackson-annotations") { because('providee ability to use jackson annotations in components') } implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" @@ -241,7 +236,6 @@ dependencies { testImplementation 'org.jetbrains.kotlin:kotlin-test-junit5' testImplementation "org.testcontainers:testcontainers:1.17.1" testImplementation "org.testcontainers:rabbitmq:1.17.1" - testImplementation "com.datastax.oss:java-driver-core:4.15.0" testFixturesImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit5', version: kotlin_version testFixturesImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 36900ff01..dd1e34b74 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -54,8 +54,8 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.MessageConverter; import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.RabbitCustomRouter; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoGroupBatch; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter; +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.GroupBatch; +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportGroupBatchRouter; import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule; import com.exactpro.th2.common.schema.util.Log4jConfigUtils; import com.fasterxml.jackson.core.JsonProcessingException; @@ -149,7 +149,7 @@ public abstract class AbstractCommonFactory implements AutoCloseable { private final AtomicReference> messageRouterParsedBatch = new AtomicReference<>(); private final AtomicReference> messageRouterRawBatch = new AtomicReference<>(); private final AtomicReference> messageRouterMessageGroupBatch = new AtomicReference<>(); - private final AtomicReference> demoMessageBatchRouter = new AtomicReference<>(); + private final AtomicReference> transportGroupBatchRouter = new AtomicReference<>(); private final AtomicReference> eventBatchRouter = new AtomicReference<>(); private final AtomicReference> notificationEventBatchRouter = new AtomicReference<>(); private final AtomicReference rootEventId = new AtomicReference<>(); @@ -295,13 +295,13 @@ public MessageRouter getMessageRouterRawBatch() { } /** - * @return Initialized {@link MessageRouter} which works with {@link DemoGroupBatch} + * @return Initialized {@link MessageRouter} which works with {@link GroupBatch} * @throws IllegalStateException if can not read configuration */ - public MessageRouter getDemoMessageBatchRouter() { - return demoMessageBatchRouter.updateAndGet(router -> { + public MessageRouter getTransportGroupBatchRouter() { + return transportGroupBatchRouter.updateAndGet(router -> { if (router == null) { - router = new DemoMessageBatchRouter(); + router = new TransportGroupBatchRouter(); router.init(getMessageRouterContext()); } @@ -532,6 +532,7 @@ public CradleManager getCradleManager() { cassandraConnectionSettings.setPassword(confidentialConfiguration.getPassword()); } + // Deserialize on config by two different beans for backward compatibility CradleNonConfidentialConfiguration nonConfidentialConfiguration = getCradleNonConfidentialConfiguration(); CassandraStorageSettings cassandraStorageSettings = getCassandraStorageSettings(); cassandraStorageSettings.setKeyspace(confidentialConfiguration.getKeyspace()); diff --git a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java index 2d0fe5964..01a9f8302 100644 --- a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java +++ b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java @@ -26,6 +26,8 @@ public abstract class AbstractTh2MsgFilterStrategy extends AbstractFilterStrategy { + public static final String BOOK_KEY = "book"; + public static final String SESSION_GROUP_KEY = "session_group"; public static final String SESSION_ALIAS_KEY = "session_alias"; public static final String MESSAGE_TYPE_KEY = "message_type"; public static final String DIRECTION_KEY = "direction"; @@ -41,8 +43,12 @@ public Map getFields(com.google.protobuf.Message message) { .map(entry -> Map.entry(entry.getKey(), entry.getValue().getSimpleValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + String sessionGroup = messageID.getConnectionId().getSessionGroup(); + String sessionAlias = messageID.getConnectionId().getSessionAlias(); var metadataMsgFields = Map.of( - SESSION_ALIAS_KEY, messageID.getConnectionId().getSessionAlias(), + BOOK_KEY, messageID.getBookName(), + SESSION_GROUP_KEY, sessionGroup.isEmpty() ? sessionAlias : sessionGroup, + SESSION_ALIAS_KEY, sessionAlias, MESSAGE_TYPE_KEY, metadata.getMessageType(), DIRECTION_KEY, messageID.getDirection().name() ); diff --git a/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt b/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt index ed4587256..0459db4d6 100644 --- a/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/metrics/MetricsUtils.kt @@ -72,7 +72,6 @@ data class SessionStats( var rawMessages: Int = 0, ) -//FIXME: com.exactpro.th2.common.metrics.MetricsUtilsKt.incrementTotalMetrics() 30,374 ms (12.5%) fun incrementTotalMetrics( batch: MessageGroupBatch, th2Pin: String, @@ -83,7 +82,6 @@ fun incrementTotalMetrics( val incomingStatsBySession = mutableMapOf() val outgoingStatsBySession = mutableMapOf() - //FIXME: java.util.Collections$UnmodifiableCollection.iterator() 14,357 ms (5.9%) for (group in batch.groupsList) { val messages = group.messagesList diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt index 17372fedf..586d91d05 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/cradle/CradleConfiguration.kt @@ -38,8 +38,8 @@ data class CradleNonConfidentialConfiguration( var cradleMaxEventBatchSize: Long = CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE.toLong(), @Deprecated("Please use CassandraStorageSettings.maxMessageBatchSize") var cradleMaxMessageBatchSize: Long = CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE.toLong(), - @Deprecated("counterPersistenceInterval") + @Deprecated("Please use CassandraStorageSettings.counterPersistenceInterval") var statisticsPersistenceIntervalMillis: Long = CassandraStorageSettings.DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS.toLong(), - @Deprecated("maxUncompressedTestEventSize") + @Deprecated("Please use CassandraStorageSettings.maxUncompressedTestEventSize") var maxUncompressedEventBatchSize: Long = CassandraStorageSettings.DEFAULT_MAX_UNCOMPRESSED_TEST_EVENT_SIZE.toLong(), ) : Configuration() \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt index d39f229d1..7d6231020 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,19 +25,27 @@ class AnyMessageFilterStrategy : AbstractFilterStrategy() { override fun getFields(message: Message): MutableMap { check(message is AnyMessage) { "Message is not an ${AnyMessage::class.qualifiedName}: ${message.toJson()}" } - val result = HashMap(); + val result = HashMap() when { message.hasMessage() -> { result.putAll(message.message.fieldsMap.mapValues { it.value.simpleValue }) val metadata = message.message.metadata - result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = metadata.id.connectionId.sessionAlias + val sessionAlias = metadata.id.connectionId.sessionAlias + val sessionGroup = metadata.id.connectionId.sessionGroup + result[AbstractTh2MsgFilterStrategy.BOOK_KEY] = metadata.id.bookName + result[AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY] = sessionGroup.ifEmpty { sessionAlias } + result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = sessionAlias result[AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY] = metadata.messageType result[AbstractTh2MsgFilterStrategy.DIRECTION_KEY] = metadata.id.direction.name } message.hasRawMessage() -> { val metadata = message.rawMessage.metadata + val sessionAlias = metadata.id.connectionId.sessionAlias + val sessionGroup = metadata.id.connectionId.sessionGroup + result[AbstractTh2MsgFilterStrategy.BOOK_KEY] = metadata.id.bookName + result[AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY] = sessionGroup.ifEmpty { sessionAlias } result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = metadata.id.connectionId.sessionAlias result[AbstractTh2MsgFilterStrategy.DIRECTION_KEY] = metadata.id.direction.name } @@ -46,4 +54,8 @@ class AnyMessageFilterStrategy : AbstractFilterStrategy() { return result } + + companion object { + val INSTANCE = AnyMessageFilterStrategy() + } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index aca845244..511ab4507 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -16,27 +16,15 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration import com.exactpro.th2.common.schema.exception.RouterException -import com.exactpro.th2.common.schema.filter.strategy.FilterStrategy -import com.exactpro.th2.common.schema.message.ConfirmationListener -import com.exactpro.th2.common.schema.message.ManualConfirmationListener -import com.exactpro.th2.common.schema.message.ExclusiveSubscriberMonitor -import com.exactpro.th2.common.schema.message.MessageListener -import com.exactpro.th2.common.schema.message.MessageRouter -import com.exactpro.th2.common.schema.message.MessageRouterContext -import com.exactpro.th2.common.schema.message.MessageSender -import com.exactpro.th2.common.schema.message.MessageSubscriber +import com.exactpro.th2.common.schema.message.* import com.exactpro.th2.common.schema.message.QueueAttribute.PUBLISH import com.exactpro.th2.common.schema.message.QueueAttribute.SUBSCRIBE -import com.exactpro.th2.common.schema.message.SubscriberMonitor -import com.exactpro.th2.common.schema.message.appendAttributes import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration -import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager -import com.google.protobuf.Message +import mu.KotlinLogging import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference -import mu.KotlinLogging typealias PinName = String typealias PinConfiguration = QueueConfiguration @@ -64,16 +52,6 @@ abstract class AbstractRabbitRouter : MessageRouter { private val subscribers = ConcurrentHashMap>() private val senders = ConcurrentHashMap>() - private val filterStrategy = AtomicReference(getDefaultFilterStrategy()) - - protected open fun getDefaultFilterStrategy(): FilterStrategy { - return FilterStrategy.DEFAULT_FILTER_STRATEGY - } - - protected open fun filterMessage(msg: Message, filters: List): Boolean { - return filterStrategy.get().verify(msg, filters) - } - override fun init(context: MessageRouterContext) { this.context = context } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt deleted file mode 100644 index 44267bc42..000000000 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/Demo.kt +++ /dev/null @@ -1,737 +0,0 @@ -/* - * Copyright 2023 Exactpro (Exactpro Systems Limited) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo - -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoDirection.INCOMING -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoDirection.OUTGOING -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessage.Companion.DEFAULT_METADATA -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.dataformat.cbor.databind.CBORMapper -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.kotlin.jacksonTypeRef -import io.netty.buffer.ByteBuf -import io.netty.buffer.ByteBufInputStream -import io.netty.buffer.ByteBufOutputStream -import io.netty.buffer.ByteBufUtil -import io.netty.buffer.Unpooled -import java.io.InputStream -import java.io.OutputStream -import java.nio.charset.Charset -import java.time.Instant -import java.util.* -import java.util.Collections.emptyList - -// TODO: maybe make length field a variable length int - -enum class ValueType(val codec: ValueCodec<*>) { - UNKNOWN(UnknownValueCodec), - LONG_TYPE(LongTypeCodec), - STRING_TYPE(StringTypeCodec), - MESSAGE_ID(MessageIdCodec), - BOOK(BookCodec), - SESSION_GROUP(SessionGroupCodec), - SESSION_ALIAS(SessionAliasCodec), - DIRECTION(DirectionCodec), - SEQUENCE(SequenceCodec), - SUBSEQUENCE(SubsequenceCodec), - TIMESTAMP(TimestampCodec), - METADATA(MetadataCodec), - PROTOCOL(ProtocolCodec), - MESSAGE_TYPE(MessageTypeCodec), - ID_CODEC(IdCodec), - SCOPE_CODEC(ScopeCodec), - EVENT_ID_CODEC(EventIdCodec), - RAW_MESSAGE(RawMessageCodec), - RAW_MESSAGE_BODY(RawMessageBodyCodec), - PARSED_MESSAGE(ParsedMessageCodec), - PARSED_MESSAGE_BODY(ParsedMessageBodyCodec), - MESSAGE_GROUP(MessageGroupCodec), - MESSAGE_LIST(MessageListCodec), - GROUP_BATCH(GroupBatchCodec), - GROUP_LIST(GroupListCodec); - - companion object { - private val MAPPING = arrayOfNulls(UByte.MAX_VALUE.toInt()).apply { - ValueType.values().forEach { this[it.codec.type.toInt()] = it } - } - - fun forId(id: UByte): ValueType = MAPPING[id.toInt()] ?: UNKNOWN - } -} - -interface ValueCodec { - val type: UByte - fun encode(source: T, target: ByteBuf) - fun decode(source: ByteBuf): T -} - -object UnknownValueCodec : ValueCodec { - override val type: UByte = 0u - override fun decode(source: ByteBuf): ByteBuf = source.readSlice(source.skipBytes(Byte.SIZE_BYTES).readIntLE()) - override fun encode(source: ByteBuf, target: ByteBuf): Nothing = throw UnsupportedOperationException() -} - -abstract class AbstractCodec(final override val type: UByte) : ValueCodec { - override fun encode(source: T, target: ByteBuf) { - val lengthIndex = target.writeByte(type.toInt()).writerIndex() - target.writeIntLE(0) - val valueIndex = target.writerIndex() - write(target, source) - target.setIntLE(lengthIndex, target.writerIndex() - valueIndex) - } - - protected abstract fun write(buffer: ByteBuf, value: T) - - override fun decode(source: ByteBuf): T { - val tag = source.readByte().toUByte() - check(tag == this.type) { "Unexpected type tag: $tag (expected: ${this.type})" } - val length = source.readIntLE() - return read(source.readSlice(length)) // FIXME: avoid slicing to avoid buffer allocation - } - - protected abstract fun read(buffer: ByteBuf): T -} - -abstract class StringCodec( - type: UByte, - private val charset: Charset = Charsets.UTF_8, -) : AbstractCodec(type) { - override fun read(buffer: ByteBuf): String = buffer.readCharSequence(buffer.readableBytes(), charset).toString() - - override fun write(buffer: ByteBuf, value: String) { - buffer.writeCharSequence(value, charset) - } -} - -abstract class LongCodec(type: UByte) : AbstractCodec(type) { - override fun read(buffer: ByteBuf): Long = buffer.readLongLE() - - override fun write(buffer: ByteBuf, value: Long) { - buffer.writeLongLE(value) - } -} - -abstract class ListCodec(type: UByte, private val elementCodec: ValueCodec) : AbstractCodec>(type) { - override fun read(buffer: ByteBuf): MutableList = mutableListOf().also { list -> - while (buffer.isReadable) { - list += elementCodec.decode(buffer) - } - } - - override fun write(buffer: ByteBuf, value: MutableList) { - value.forEach { elementCodec.encode(it, buffer) } - } -} - -abstract class MapCodec( - type: UByte, - private val keyCodec: ValueCodec, - private val valueCodec: ValueCodec, -) : AbstractCodec>(type) { - override fun read(buffer: ByteBuf): MutableMap = mutableMapOf().apply { - while (buffer.isReadable) { - this[keyCodec.decode(buffer)] = valueCodec.decode(buffer) - } - } - - override fun write(buffer: ByteBuf, value: MutableMap): Unit = value.forEach { (key, value) -> - keyCodec.encode(key, buffer) - valueCodec.encode(value, buffer) - } -} - -abstract class ByteBufCodec(type: UByte) : AbstractCodec(type) { - override fun read(buffer: ByteBuf): ByteBuf = buffer.copy() - - override fun write(buffer: ByteBuf, value: ByteBuf) { - value.markReaderIndex().apply(buffer::writeBytes).resetReaderIndex() - } -} - -abstract class InstantCodec(type: UByte) : AbstractCodec(type) { - override fun read(buffer: ByteBuf): Instant = Instant.ofEpochSecond(buffer.readLongLE(), buffer.readIntLE().toLong()) - - override fun write(buffer: ByteBuf, value: Instant) { - buffer.writeLongLE(value.epochSecond).writeIntLE(value.nano) - } -} - -open class CborCodec(type: UByte, private val typeReference: TypeReference) : AbstractCodec(type) { - override fun read(buffer: ByteBuf): T = ByteBufInputStream(buffer).use { - return MAPPER.readValue(it as InputStream, typeReference) - } - - override fun write(buffer: ByteBuf, value: T) = ByteBufOutputStream(buffer).use { - MAPPER.writeValue(it as OutputStream, value) - } - - companion object { - private val MAPPER = CBORMapper().registerModule(JavaTimeModule()) - } -} - -object LongTypeCodec : LongCodec(1u) - -object StringTypeCodec : StringCodec(2u) - -object MessageIdCodec : AbstractCodec(10u) { - override fun read(buffer: ByteBuf): DemoMessageId = DemoMessageId().apply { - buffer.forEachValue { codec -> - when (codec) { - is SessionAliasCodec -> sessionAlias = codec.decode(buffer) - is DirectionCodec -> direction = codec.decode(buffer) - is SequenceCodec -> sequence = codec.decode(buffer) - is SubsequenceCodec -> subsequence = codec.decode(buffer) - is TimestampCodec -> timestamp = codec.decode(buffer) - else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") - } - } - } - - override fun write(buffer: ByteBuf, value: DemoMessageId) { - SessionAliasCodec.encode(value.sessionAlias, buffer) - DirectionCodec.encode(value.direction, buffer) - SequenceCodec.encode(value.sequence, buffer) - SubsequenceCodec.encode(value.subsequence, buffer) - TimestampCodec.encode(value.timestamp, buffer) - } -} - -object BookCodec : StringCodec(101u) - -object SessionGroupCodec : StringCodec(102u) - -object SessionAliasCodec : StringCodec(103u) - -object DirectionCodec : AbstractCodec(104u) { - override fun read(buffer: ByteBuf): DemoDirection = DemoDirection.forId(buffer.readByte().toInt()) - - override fun write(buffer: ByteBuf, value: DemoDirection) { - buffer.writeByte(value.id) - } -} - -object SequenceCodec : LongCodec(105u) - -object SubsequenceCodec : ListCodec(106u, LongTypeCodec) - -object TimestampCodec : InstantCodec(107u) - -object MetadataCodec : MapCodec(11u, StringTypeCodec, StringTypeCodec) - -object ProtocolCodec : StringCodec(12u) - -object MessageTypeCodec : StringCodec(13u) - -object IdCodec : StringCodec(14u) - -object ScopeCodec : StringCodec(15u) - -object EventIdCodec : AbstractCodec(16u) { - override fun read(buffer: ByteBuf): DemoEventId = DemoEventId().apply { - buffer.forEachValue { codec -> - when (codec) { - is IdCodec -> id = codec.decode(buffer) - is BookCodec -> book = codec.decode(buffer) - is ScopeCodec -> scope = codec.decode(buffer) - is TimestampCodec -> timestamp = codec.decode(buffer) - } - } - } - - override fun write(buffer: ByteBuf, value: DemoEventId) { - IdCodec.encode(value.id, buffer) - BookCodec.encode(value.book, buffer) - ScopeCodec.encode(value.scope, buffer) - TimestampCodec.encode(value.timestamp, buffer) - } -} - -object RawMessageCodec : AbstractCodec(20u) { - override fun read(buffer: ByteBuf): DemoRawMessage = DemoRawMessage().apply { - buffer.forEachValue { codec -> - when (codec) { - is MessageIdCodec -> id = codec.decode(buffer) - is EventIdCodec -> eventId = codec.decode(buffer) - is MetadataCodec -> metadata = codec.decode(buffer) - is ProtocolCodec -> protocol = codec.decode(buffer) - is RawMessageBodyCodec -> body = codec.decode(buffer) - else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") - } - } - } - - override fun write(buffer: ByteBuf, value: DemoRawMessage) { - MessageIdCodec.encode(value.id, buffer) - value.eventId?.run { EventIdCodec.encode(this, buffer) } - MetadataCodec.encode(value.metadata, buffer) - ProtocolCodec.encode(value.protocol, buffer) - RawMessageBodyCodec.encode(value.body, buffer) - } -} - -object RawMessageBodyCodec : ByteBufCodec(21u) - -object ParsedMessageCodec : AbstractCodec(30u) { - override fun read(buffer: ByteBuf): DemoParsedMessage = DemoParsedMessage().apply { - buffer.forEachValue { codec -> - when (codec) { - is MessageIdCodec -> id = codec.decode(buffer) - is EventIdCodec -> eventId = codec.decode(buffer) - is MetadataCodec -> metadata = codec.decode(buffer) - is ProtocolCodec -> protocol = codec.decode(buffer) - is MessageTypeCodec -> type = codec.decode(buffer) - is ParsedMessageBodyCodec -> body = codec.decode(buffer) - else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") - } - } - } - - override fun write(buffer: ByteBuf, value: DemoParsedMessage) { - MessageIdCodec.encode(value.id, buffer) - value.eventId?.run { EventIdCodec.encode(this, buffer) } - MetadataCodec.encode(value.metadata, buffer) - ProtocolCodec.encode(value.protocol, buffer) - MessageTypeCodec.encode(value.type, buffer) - ParsedMessageBodyCodec.encode(value.body, buffer) - } -} - -object ParsedMessageBodyCodec : CborCodec>(31u, jacksonTypeRef()) - -object MessageGroupCodec : AbstractCodec(40u) { - override fun read(buffer: ByteBuf): DemoMessageGroup = DemoMessageGroup().apply { - buffer.forEachValue { codec -> - when (codec) { - is MessageListCodec -> messages = codec.decode(buffer) - else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") - } - } - } - - override fun write(buffer: ByteBuf, value: DemoMessageGroup) { - MessageListCodec.encode(value.messages, buffer) - } -} - -object MessageListCodec : AbstractCodec>>(41u) { - override fun read(buffer: ByteBuf): MutableList> = mutableListOf>().apply { - buffer.forEachValue { codec -> - when (codec) { - is RawMessageCodec -> this += codec.decode(buffer) - is ParsedMessageCodec -> this += codec.decode(buffer) - else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") - } - } - } - - override fun write(buffer: ByteBuf, value: MutableList>): Unit = value.forEach { message -> - when (message) { - is DemoRawMessage -> RawMessageCodec.encode(message, buffer) - is DemoParsedMessage -> ParsedMessageCodec.encode(message, buffer) - else -> println("Skipping unsupported message type: $message") - } - } -} - -object GroupBatchCodec : AbstractCodec(50u) { - override fun read(buffer: ByteBuf): DemoGroupBatch = DemoGroupBatch().apply { - buffer.forEachValue { codec -> - when (codec) { - is BookCodec -> book = codec.decode(buffer) - is SessionGroupCodec -> sessionGroup = codec.decode(buffer) - is GroupListCodec -> groups = codec.decode(buffer) - else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") - } - } - - groups.forEach { group -> - group.messages.forEach { message -> - val id = message.id - id.book = book - id.sessionGroup = sessionGroup - } - } - } - - override fun write(buffer: ByteBuf, value: DemoGroupBatch) { - BookCodec.encode(value.book, buffer) - SessionGroupCodec.encode(value.sessionGroup, buffer) - GroupListCodec.encode(value.groups, buffer) - } -} - -object GroupListCodec : ListCodec(51u, MessageGroupCodec) - -inline fun ByteBuf.forEachValue(action: (codec: ValueCodec<*>) -> Unit) { - while (isReadable) { - val type = getByte(readerIndex()).toUByte() - - when (val codec = ValueType.forId(type).codec) { - is UnknownValueCodec -> println("Skipping unknown type $type value: ${ByteBufUtil.hexDump(codec.decode(this))}") - else -> action(codec) - } - } -} - -enum class DemoDirection(val id: Int) { - INCOMING(1), - OUTGOING(2); - - companion object { - fun forId(id: Int): DemoDirection = when (id) { - 1 -> INCOMING - 2 -> OUTGOING - else -> error("Unknown direction id: $id") - } - } -} - -interface Cleanable { - /** - * Cleans all embedded th2 objects - */ - fun clean() - - /** - * Cleans only current layer include collections. - */ - fun softClean() = clean() -} - -data class DemoMessageId( - var book: String = "", - var sessionGroup: String = "", - var sessionAlias: String = "", - var direction: DemoDirection = INCOMING, - var sequence: Long = 0, - /** The subsequence is not mutable by default */ - var subsequence: MutableList = DEFAULT_SUBSEQUENCE, - var timestamp: Instant = Instant.EPOCH, -) : Cleanable { - - override fun clean() { - check(this !== DEFAULT_INSTANCE) { - "Object can be cleaned because it is default instance" - } - check(subsequence !== DEFAULT_SUBSEQUENCE) { - "Object can be cleaned because 'subsequence' is immutable" - } - - book = "" - sessionGroup = "" - sessionAlias = "" - direction = INCOMING - sequence = 0 - subsequence.clear() - timestamp = Instant.EPOCH - } - - companion object { - val DEFAULT_SUBSEQUENCE: MutableList = emptyList() - val DEFAULT_INSTANCE: DemoMessageId = DemoMessageId() // FIXME: do smth about its mutability - fun newMutable() = DemoMessageId( - subsequence = mutableListOf() - ) - } -} - -data class DemoEventId( - var id: String = "", - var book: String = "", - var scope: String = "", - var timestamp: Instant = Instant.EPOCH, -) : Cleanable { - - override fun clean() { - check(this !== DEFAULT_INSTANCE) { - "Object can be cleaned because it is default instance" - } - id= "" - book = "" - scope = "" - timestamp = Instant.EPOCH - } - - companion object { - val DEFAULT_INSTANCE: DemoEventId = DemoEventId() // FIXME: do smth about its mutability - fun newMutable() = DemoEventId() - } -} - -interface DemoMessage : Cleanable { - /** The id is not mutable by default */ - var id: DemoMessageId - var eventId: DemoEventId? - /** The metadata is not mutable by default */ - var metadata: MutableMap - var protocol: String - var body: T - - companion object { - val DEFAULT_METADATA: MutableMap = Collections.emptyMap() - } -} - -data class DemoRawMessage( - override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, - override var eventId: DemoEventId? = null, - override var metadata: MutableMap = DEFAULT_METADATA, - override var protocol: String = "", - /** The body is not mutable by default */ - override var body: ByteBuf = Unpooled.EMPTY_BUFFER, -) : DemoMessage { - override fun clean() { - check(id !== DemoMessageId.DEFAULT_INSTANCE) { - "Object can be cleaned because 'id' is default instance" - } - check(metadata !== DEFAULT_METADATA) { - "Object can be cleaned because 'metadata' is immutable" - } - check(body !== Unpooled.EMPTY_BUFFER) { - "Object can be cleaned because 'body' is immutable" - } - - id.clean() - eventId = null - metadata.clear() - protocol = "" - body.clear() - } - - override fun softClean() { - check(metadata !== DEFAULT_METADATA) { - "Object can be cleaned because 'metadata' is immutable" - } - - id = DemoMessageId.DEFAULT_INSTANCE - eventId = null - metadata.clear() - protocol = "" - body = Unpooled.EMPTY_BUFFER - } - - companion object { - fun newMutable() = DemoRawMessage( - id = DemoMessageId.newMutable(), - metadata = mutableMapOf(), - body = Unpooled.buffer() - ) - fun newSoftMutable() = DemoRawMessage( - metadata = mutableMapOf() - ) - } -} - -data class DemoParsedMessage( - override var id: DemoMessageId = DemoMessageId.DEFAULT_INSTANCE, - override var eventId: DemoEventId? = null, - override var metadata: MutableMap = DEFAULT_METADATA, - override var protocol: String = "", - var type: String = "", - /** The body is not mutable by default */ - override var body: MutableMap = DEFAULT_BODY, -) : DemoMessage> { - override fun clean() { - check(id !== DemoMessageId.DEFAULT_INSTANCE) { - "Object can be cleaned because 'id' is default instance" - } - check(metadata !== DEFAULT_METADATA) { - "Object can be cleaned because 'metadata' is immutable" - } - check(body !== DEFAULT_BODY) { - "Object can be cleaned because 'body' is immutable" - } - - id.clean() - eventId = null - metadata.clear() - protocol = "" - type = "" - body.clear() - } - - override fun softClean() { - check(metadata !== DEFAULT_METADATA) { - "Object can be cleaned because 'metadata' is immutable" - } - check(body !== DEFAULT_BODY) { - "Object can be cleaned because 'body' is immutable" - } - - id = DemoMessageId.DEFAULT_INSTANCE - eventId = null - metadata.clear() - protocol = "" - type = "" - body.clear() - } - - companion object { - val DEFAULT_BODY: MutableMap = Collections.emptyMap() - fun newMutable() = DemoParsedMessage( - id = DemoMessageId.newMutable(), - metadata = mutableMapOf(), - body = mutableMapOf() - ) - fun newSoftMutable() = DemoParsedMessage( - metadata = mutableMapOf(), - body = mutableMapOf() - ) - } -} - -data class DemoMessageGroup( - var messages: MutableList> = DEFAULT_MESSAGES, -) : Cleanable { - override fun clean() { - check(messages !== DEFAULT_MESSAGES) { - "Object can be cleaned because 'messages' is immutable" - } - messages.clear() - } - - companion object { - val DEFAULT_MESSAGES: MutableList> = emptyList() - fun newMutable() = DemoMessageGroup(mutableListOf()) - } -} - -data class DemoGroupBatch( - var book: String = "", - var sessionGroup: String = "", - var groups: MutableList = DEFAULT_GROUPS, -) : Cleanable { - override fun clean() { - check(groups !== DEFAULT_GROUPS) { - "Object can be cleaned because 'groups' is immutable" - } - - book = "" - sessionGroup = "" - groups.clear() - } - - companion object { - val DEFAULT_GROUPS: MutableList = emptyList() - fun newMutable() = DemoGroupBatch( - groups = mutableListOf() - ) - } -} - -fun DemoGroupBatch.toByteArray() = Unpooled.buffer().run { - GroupBatchCodec.encode(this@toByteArray, this@run) - ByteArray(readableBytes()).apply(::readBytes) -} - -fun main() { - DemoMessageId.newMutable().apply(DemoMessageId::clean) - .apply(DemoMessageId::softClean) - DemoEventId.newMutable().apply(DemoEventId::clean) - .apply(DemoEventId::softClean) - - DemoRawMessage.newMutable().apply(DemoRawMessage::clean) - .apply(DemoRawMessage::softClean) - DemoRawMessage.newSoftMutable().apply(DemoRawMessage::softClean) - - DemoParsedMessage.newMutable().apply(DemoParsedMessage::clean) - .apply(DemoParsedMessage::softClean) - DemoParsedMessage.newSoftMutable().apply(DemoParsedMessage::softClean) - - DemoMessageGroup.newMutable().apply(DemoMessageGroup::clean) - .apply(DemoMessageGroup::softClean) - - DemoGroupBatch.newMutable().apply(DemoGroupBatch::clean) - .apply(DemoGroupBatch::softClean) - - val buffer = Unpooled.buffer() - - val message1 = DemoRawMessage( - id = DemoMessageId( - book = "book1", - sessionGroup = "group1", - sessionAlias = "alias1", - direction = INCOMING, - sequence = 1, - subsequence = mutableListOf(1, 2), - timestamp = Instant.now() - ), - metadata = mutableMapOf( - "prop1" to "value1", - "prop2" to "value2" - ), - protocol = "proto1", - body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3, 4)) - ) - - val message2 = DemoRawMessage( - id = DemoMessageId( - book = "book1", - sessionGroup = "group1", - sessionAlias = "alias2", - direction = OUTGOING, - sequence = 2, - subsequence = mutableListOf(3, 4), - timestamp = Instant.now() - ), - metadata = mutableMapOf( - "prop3" to "value3", - "prop4" to "value4" - ), - protocol = "proto2", - body = Unpooled.wrappedBuffer(byteArrayOf(5, 6, 7, 8)) - ) - - val message3 = DemoParsedMessage( - id = DemoMessageId( - book = "book1", - sessionGroup = "group1", - sessionAlias = "alias3", - direction = OUTGOING, - sequence = 3, - subsequence = mutableListOf(5, 6), - timestamp = Instant.now() - ), - metadata = mutableMapOf( - "prop5" to "value6", - "prop7" to "value8" - ), - protocol = "proto3", - type = "some-type", - body = mutableMapOf( - "simple" to 1, - "list" to listOf(1, 2, 3), - "map" to mapOf("abc" to "cde") - ) - ) - - val batch = DemoGroupBatch( - book = "book1", - sessionGroup = "group1", - groups = mutableListOf(DemoMessageGroup(mutableListOf(message1, message2, message3))) - ) - - GroupBatchCodec.encode(batch, buffer) - - val decodedBatch = GroupBatchCodec.decode(buffer) - - println(batch) - println(decodedBatch) - println(batch == decodedBatch) - println(buffer) -} diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt index 54da8c53e..1cf80c7d8 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -16,13 +16,8 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.group import com.exactpro.th2.common.grpc.MessageGroupBatch -import com.exactpro.th2.common.metrics.DIRECTION_LABEL -import com.exactpro.th2.common.metrics.MESSAGE_TYPE_LABEL -import com.exactpro.th2.common.metrics.SESSION_ALIAS_LABEL -import com.exactpro.th2.common.metrics.TH2_PIN_LABEL -import com.exactpro.th2.common.metrics.incrementDroppedMetrics -import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractFilterStrategy -import com.exactpro.th2.common.schema.filter.strategy.impl.AnyMessageFilterStrategy +import com.exactpro.th2.common.metrics.* +import com.exactpro.th2.common.schema.filter.strategy.impl.AnyMessageFilterStrategy.Companion.INSTANCE import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration @@ -37,9 +32,6 @@ import io.prometheus.client.Counter import org.jetbrains.annotations.NotNull class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() { - override fun getDefaultFilterStrategy(): AbstractFilterStrategy { - return AnyMessageFilterStrategy() - } override fun splitAndFilter( message: MessageGroupBatch, @@ -52,7 +44,7 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() val builder = message.toBuilderWithMetadata() message.groupsList.forEach { group -> - if (group.messagesList.all { filterMessage(it, pinConfiguration.filters) }) { + if (group.messagesList.all { INSTANCE.verify(it, pinConfiguration.filters) }) { builder.addGroups(group) } else { incrementDroppedMetrics( @@ -88,7 +80,7 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() return RabbitMessageGroupBatchSubscriber( connectionManager, pinConfig.queue, - { msg: Message, filters: List -> filterMessage(msg, filters) }, + { msg: Message, filters: List -> INSTANCE.verify(msg, filters) }, pinName, pinConfig.filters, connectionManager.configuration.messageRecursionLimit diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt index d80e80cdc..fcffae922 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Cleanable.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Cleanable.kt new file mode 100644 index 000000000..df55289b5 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Cleanable.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +interface Cleanable { + /** + * Deep clean + */ + fun clean() + + /** + * Cleans only current layer of th2 entity include collections. + * This method can be useful when you need th2 entity builder, but you are going to operate th2 entity separately + */ + fun softClean() = clean() +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt new file mode 100644 index 000000000..712122f9d --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt @@ -0,0 +1,407 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.dataformat.cbor.databind.CBORMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.jacksonTypeRef +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufInputStream +import io.netty.buffer.ByteBufOutputStream +import io.netty.buffer.ByteBufUtil +import java.io.InputStream +import java.io.OutputStream +import java.nio.charset.Charset +import java.time.Instant + +// TODO: maybe make length field a variable length int + +@Suppress("unused") +enum class ValueType(val codec: ValueCodec<*>) { + UNKNOWN(UnknownValueCodec), + LONG_TYPE(LongTypeCodec), + STRING_TYPE(StringTypeCodec), + MESSAGE_ID(MessageIdCodec), + BOOK(BookCodec), + SESSION_GROUP(SessionGroupCodec), + SESSION_ALIAS(SessionAliasCodec), + DIRECTION(DirectionCodec), + SEQUENCE(SequenceCodec), + SUBSEQUENCE(SubsequenceCodec), + TIMESTAMP(TimestampCodec), + METADATA(MetadataCodec), + PROTOCOL(ProtocolCodec), + MESSAGE_TYPE(MessageTypeCodec), + ID_CODEC(IdCodec), + SCOPE_CODEC(ScopeCodec), + EVENT_ID_CODEC(EventIdCodec), + RAW_MESSAGE(RawMessageCodec), + RAW_MESSAGE_BODY(RawMessageBodyCodec), + PARSED_MESSAGE(ParsedMessageCodec), + PARSED_MESSAGE_BODY(ParsedMessageBodyCodec), + MESSAGE_GROUP(MessageGroupCodec), + MESSAGE_LIST(MessageListCodec), + GROUP_BATCH(GroupBatchCodec), + GROUP_LIST(GroupListCodec); + + companion object { + private val MAPPING: Array = arrayOfNulls(UByte.MAX_VALUE.toInt()).apply { + ValueType.values().forEach { + this[it.codec.type.toInt()]?.let { previous -> + error("$previous and $it elements of ValueType enum have the same type byte - ${it.codec.type}") + } + this[it.codec.type.toInt()] = it + } + } + + fun forId(id: UByte): ValueType = MAPPING[id.toInt()] ?: UNKNOWN + } +} + +sealed interface ValueCodec { + val type: UByte + fun encode(source: T, target: ByteBuf) + fun decode(source: ByteBuf): T +} + +object UnknownValueCodec : ValueCodec { + override val type: UByte = 0u + override fun decode(source: ByteBuf): ByteBuf = source.readSlice(source.skipBytes(Byte.SIZE_BYTES).readIntLE()) + override fun encode(source: ByteBuf, target: ByteBuf): Nothing = throw UnsupportedOperationException() +} + +abstract class AbstractCodec(final override val type: UByte) : ValueCodec { + override fun encode(source: T, target: ByteBuf) { + val lengthIndex = target.writeByte(type.toInt()).writerIndex() + target.writeIntLE(0) + val valueIndex = target.writerIndex() + write(target, source) + target.setIntLE(lengthIndex, target.writerIndex() - valueIndex) + } + + protected abstract fun write(buffer: ByteBuf, value: T) + + override fun decode(source: ByteBuf): T { + val tag = source.readByte().toUByte() + check(tag == this.type) { "Unexpected type tag: $tag (expected: ${this.type})" } + val length = source.readIntLE() + return read(source.readSlice(length)) // FIXME: avoid slicing to avoid buffer allocation + } + + protected abstract fun read(buffer: ByteBuf): T +} + +abstract class StringCodec( + type: UByte, + private val charset: Charset = Charsets.UTF_8, +) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): String = buffer.readCharSequence(buffer.readableBytes(), charset).toString() + + override fun write(buffer: ByteBuf, value: String) { + buffer.writeCharSequence(value, charset) + } +} + +abstract class LongCodec(type: UByte) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): Long = buffer.readLongLE() + + override fun write(buffer: ByteBuf, value: Long) { + buffer.writeLongLE(value) + } +} + +abstract class IntCodec(type: UByte) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): Int = buffer.readIntLE() + + override fun write(buffer: ByteBuf, value: Int) { + buffer.writeIntLE(value) + } +} + +abstract class ListCodec(type: UByte, private val elementCodec: ValueCodec) : AbstractCodec>(type) { + override fun read(buffer: ByteBuf): MutableList = mutableListOf().also { list -> + while (buffer.isReadable) { + list += elementCodec.decode(buffer) + } + } + + override fun write(buffer: ByteBuf, value: MutableList) { + value.forEach { elementCodec.encode(it, buffer) } + } +} + +abstract class MapCodec( + type: UByte, + private val keyCodec: ValueCodec, + private val valueCodec: ValueCodec, +) : AbstractCodec>(type) { + override fun read(buffer: ByteBuf): MutableMap = mutableMapOf().apply { + while (buffer.isReadable) { + this[keyCodec.decode(buffer)] = valueCodec.decode(buffer) + } + } + + override fun write(buffer: ByteBuf, value: MutableMap): Unit = value.forEach { (key, value) -> + keyCodec.encode(key, buffer) + valueCodec.encode(value, buffer) + } +} + +abstract class ByteBufCodec(type: UByte) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): ByteBuf = buffer.copy() + + override fun write(buffer: ByteBuf, value: ByteBuf) { + value.markReaderIndex().apply(buffer::writeBytes).resetReaderIndex() + } +} + +abstract class InstantCodec(type: UByte) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): Instant = Instant.ofEpochSecond(buffer.readLongLE(), buffer.readIntLE().toLong()) + + override fun write(buffer: ByteBuf, value: Instant) { + buffer.writeLongLE(value.epochSecond).writeIntLE(value.nano) + } +} + +open class CborCodec(type: UByte, private val typeReference: TypeReference) : AbstractCodec(type) { + override fun read(buffer: ByteBuf): T = ByteBufInputStream(buffer).use { + return MAPPER.readValue(it as InputStream, typeReference) + } + + override fun write(buffer: ByteBuf, value: T) = ByteBufOutputStream(buffer).use { + MAPPER.writeValue(it as OutputStream, value) + } + + companion object { + private val MAPPER = CBORMapper().registerModule(JavaTimeModule()) + } +} + +// FIXME: think about checking that type is unique +object LongTypeCodec : LongCodec(1u) + +object StringTypeCodec : StringCodec(2u) + +object IntTypeCodec : IntCodec(3u) + +object MessageIdCodec : AbstractCodec(10u) { + override fun read(buffer: ByteBuf): MessageId = MessageId().apply { + buffer.forEachValue { codec -> + when (codec) { + is SessionAliasCodec -> sessionAlias = codec.decode(buffer) + is DirectionCodec -> direction = codec.decode(buffer) + is SequenceCodec -> sequence = codec.decode(buffer) + is SubsequenceCodec -> subsequence = codec.decode(buffer) + is TimestampCodec -> timestamp = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: MessageId) { + SessionAliasCodec.encode(value.sessionAlias, buffer) + DirectionCodec.encode(value.direction, buffer) + SequenceCodec.encode(value.sequence, buffer) + SubsequenceCodec.encode(value.subsequence, buffer) + TimestampCodec.encode(value.timestamp, buffer) + } +} + +object BookCodec : StringCodec(101u) + +object SessionGroupCodec : StringCodec(102u) + +object SessionAliasCodec : StringCodec(103u) + +object DirectionCodec : AbstractCodec(104u) { + override fun read(buffer: ByteBuf): Direction = Direction.forId(buffer.readByte().toInt()) + + override fun write(buffer: ByteBuf, value: Direction) { + buffer.writeByte(value.id) + } +} + +object SequenceCodec : LongCodec(105u) + +object SubsequenceCodec : ListCodec(106u, IntTypeCodec) + +object TimestampCodec : InstantCodec(107u) + +object MetadataCodec : MapCodec(11u, StringTypeCodec, StringTypeCodec) + +object ProtocolCodec : StringCodec(12u) + +object MessageTypeCodec : StringCodec(13u) + +object IdCodec : StringCodec(14u) + +object ScopeCodec : StringCodec(15u) + +object EventIdCodec : AbstractCodec(16u) { + override fun read(buffer: ByteBuf): EventId { + var id = "" + var book = "" + var scope = "" + var timestamp: Instant = Instant.EPOCH + + buffer.forEachValue { codec -> + when (codec) { + is IdCodec -> id = codec.decode(buffer) + is BookCodec -> book = codec.decode(buffer) + is ScopeCodec -> scope = codec.decode(buffer) + is TimestampCodec -> timestamp = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + return EventId(id, book, scope, timestamp) + } + + override fun write(buffer: ByteBuf, value: EventId) { + IdCodec.encode(value.id, buffer) + BookCodec.encode(value.book, buffer) + ScopeCodec.encode(value.scope, buffer) + TimestampCodec.encode(value.timestamp, buffer) + } +} + +object RawMessageCodec : AbstractCodec(20u) { + override fun read(buffer: ByteBuf): RawMessage = RawMessage().apply { + buffer.forEachValue { codec -> + when (codec) { + is MessageIdCodec -> id = codec.decode(buffer) + is EventIdCodec -> eventId = codec.decode(buffer) + is MetadataCodec -> metadata = codec.decode(buffer) + is ProtocolCodec -> protocol = codec.decode(buffer) + is RawMessageBodyCodec -> body = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: RawMessage) { + MessageIdCodec.encode(value.id, buffer) + value.eventId?.run { EventIdCodec.encode(this, buffer) } + MetadataCodec.encode(value.metadata, buffer) + ProtocolCodec.encode(value.protocol, buffer) + RawMessageBodyCodec.encode(value.body, buffer) + } +} + +object RawMessageBodyCodec : ByteBufCodec(21u) + +object ParsedMessageCodec : AbstractCodec(30u) { + override fun read(buffer: ByteBuf): ParsedMessage = ParsedMessage().apply { + buffer.forEachValue { codec -> + when (codec) { + is MessageIdCodec -> id = codec.decode(buffer) + is EventIdCodec -> eventId = codec.decode(buffer) + is MetadataCodec -> metadata = codec.decode(buffer) + is ProtocolCodec -> protocol = codec.decode(buffer) + is MessageTypeCodec -> type = codec.decode(buffer) + is ParsedMessageBodyCodec -> body = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: ParsedMessage) { + MessageIdCodec.encode(value.id, buffer) + value.eventId?.run { EventIdCodec.encode(this, buffer) } + MetadataCodec.encode(value.metadata, buffer) + ProtocolCodec.encode(value.protocol, buffer) + MessageTypeCodec.encode(value.type, buffer) + ParsedMessageBodyCodec.encode(value.body, buffer) + } +} + +object ParsedMessageBodyCodec : CborCodec>(31u, jacksonTypeRef()) + +object MessageGroupCodec : AbstractCodec(40u) { + override fun read(buffer: ByteBuf): MessageGroup = MessageGroup().apply { + buffer.forEachValue { codec -> + when (codec) { + is MessageListCodec -> messages = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: MessageGroup) { + MessageListCodec.encode(value.messages, buffer) + } +} + +object MessageListCodec : AbstractCodec>>(41u) { + override fun read(buffer: ByteBuf): MutableList> = mutableListOf>().apply { + buffer.forEachValue { codec -> + when (codec) { + is RawMessageCodec -> this += codec.decode(buffer) + is ParsedMessageCodec -> this += codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + } + + override fun write(buffer: ByteBuf, value: MutableList>): Unit = value.forEach { message -> + when (message) { + is RawMessage -> RawMessageCodec.encode(message, buffer) + is ParsedMessage -> ParsedMessageCodec.encode(message, buffer) + else -> println("Skipping unsupported message type: $message") + } + } +} + +object GroupBatchCodec : AbstractCodec(50u) { + override fun read(buffer: ByteBuf): GroupBatch = GroupBatch().apply { + buffer.forEachValue { codec -> + when (codec) { + is BookCodec -> book = codec.decode(buffer) + is SessionGroupCodec -> sessionGroup = codec.decode(buffer) + is GroupListCodec -> groups = codec.decode(buffer) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } + } + + groups.forEach { group -> + group.messages.forEach { message -> + val id = message.id + id.book = book + id.sessionGroup = sessionGroup + } + } + } + + override fun write(buffer: ByteBuf, value: GroupBatch) { + BookCodec.encode(value.book, buffer) + SessionGroupCodec.encode(value.sessionGroup, buffer) + GroupListCodec.encode(value.groups, buffer) + } +} + +object GroupListCodec : ListCodec(51u, MessageGroupCodec) + +inline fun ByteBuf.forEachValue(action: (codec: ValueCodec<*>) -> Unit) { + while (isReadable) { + val type = getByte(readerIndex()).toUByte() + + when (val codec = ValueType.forId(type).codec) { + is UnknownValueCodec -> println("Skipping unknown type $type value: ${ByteBufUtil.hexDump(codec.decode(this))}") + else -> action(codec) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Direction.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Direction.kt new file mode 100644 index 000000000..fc68922e2 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Direction.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +enum class Direction(val id: Int) { + /** + * Related to messages are income to a client + */ + INCOMING(1), + /** + * Related to messages are out gone from a client + */ + OUTGOING(2); + + companion object { + fun forId(id: Int): Direction = when (id) { + 1 -> INCOMING + 2 -> OUTGOING + else -> error("Unknown direction id: $id") + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt new file mode 100644 index 000000000..b6afc0606 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import java.time.Instant + +data class EventId( + var id: String = "", + var book: String = "", + var scope: String = "", + var timestamp: Instant = Instant.EPOCH, +) : Cleanable { + + override fun clean() { + check(this !== DEFAULT_INSTANCE) { + "Object can be cleaned because it is default instance" + } + id= "" + book = "" + scope = "" + timestamp = Instant.EPOCH + } + + companion object { + val DEFAULT_INSTANCE: EventId = EventId() // FIXME: do smth about its mutability + fun newMutable() = EventId() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt new file mode 100644 index 000000000..4d1da7479 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import java.util.* + +data class GroupBatch( + var book: String = "", + var sessionGroup: String = "", + var groups: MutableList = DEFAULT_GROUPS, +) : Cleanable { + override fun clean() { + check(groups !== DEFAULT_GROUPS) { + "Object can be cleaned because 'groups' is immutable" + } + + groups.clear() + } + + companion object { + val DEFAULT_GROUPS: MutableList = Collections.emptyList() + fun newMutable() = GroupBatch( + groups = mutableListOf() + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt new file mode 100644 index 000000000..cd452f010 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import java.util.* + +interface Message : Cleanable { + /** The id is not mutable by default */ + var id: MessageId + var eventId: EventId? + /** The metadata is not mutable by default */ + var metadata: MutableMap + var protocol: String + var body: T + + companion object { + val DEFAULT_METADATA: MutableMap = Collections.emptyMap() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt new file mode 100644 index 000000000..2ba360c84 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import java.util.* + +data class MessageGroup( + var messages: MutableList> = DEFAULT_MESSAGES, // FIXME: message can have incompatible book and group +) : Cleanable { + override fun clean() { + check(messages !== DEFAULT_MESSAGES) { + "Object can be cleaned because 'messages' is immutable" + } + messages.clear() + } + + companion object { + val DEFAULT_MESSAGES: MutableList> = Collections.emptyList() + fun newMutable() = MessageGroup(mutableListOf()) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt new file mode 100644 index 000000000..272e9e595 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import java.time.Instant +import java.util.* + +data class MessageId( + var book: String = "", + var sessionGroup: String = "", + var sessionAlias: String = "", + var direction: Direction = Direction.INCOMING, + var sequence: Long = 0, + /** The subsequence is not mutable by default */ + var subsequence: MutableList = DEFAULT_SUBSEQUENCE, + var timestamp: Instant = Instant.EPOCH, +) : Cleanable { + override fun clean() { + check(this !== DEFAULT_INSTANCE) { + "Object can be cleaned because it is default instance" + } + check(subsequence !== DEFAULT_SUBSEQUENCE) { + "Object can be cleaned because 'subsequence' is immutable" + } + + book = "" + sessionGroup = "" + sessionAlias = "" + direction = Direction.INCOMING + sequence = 0 + subsequence.clear() + timestamp = Instant.EPOCH + } + + companion object { + val DEFAULT_SUBSEQUENCE: MutableList = Collections.emptyList() + val DEFAULT_INSTANCE: MessageId = MessageId() // FIXME: do smth about its mutability + fun newMutable() = MessageId( + subsequence = mutableListOf() + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt new file mode 100644 index 000000000..5e58601ee --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt @@ -0,0 +1,77 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import java.util.* + +data class ParsedMessage( + override var id: MessageId = MessageId.DEFAULT_INSTANCE, + override var eventId: EventId? = null, + override var metadata: MutableMap = Message.DEFAULT_METADATA, + override var protocol: String = "", + var type: String = "", + /** The body is not mutable by default */ + override var body: MutableMap = DEFAULT_BODY, // FIXME: should be lazy deserializing +) : Message> { + override fun clean() { + check(id !== MessageId.DEFAULT_INSTANCE) { + "Object can be cleaned because 'id' is default instance" + } + check(metadata !== Message.DEFAULT_METADATA) { + "Object can be cleaned because 'metadata' is immutable" + } + check(body !== DEFAULT_BODY) { + "Object can be cleaned because 'body' is immutable" + } + + id.clean() + eventId = null + metadata.clear() + protocol = "" + type = "" + body.clear() + } + + override fun softClean() { + check(metadata !== Message.DEFAULT_METADATA) { + "Object can be cleaned because 'metadata' is immutable" + } + check(body !== DEFAULT_BODY) { + "Object can be cleaned because 'body' is immutable" + } + + id = MessageId.DEFAULT_INSTANCE + eventId = null + metadata.clear() + protocol = "" + type = "" + body.clear() + } + + companion object { + val DEFAULT_BODY: MutableMap = Collections.emptyMap() + fun newMutable() = ParsedMessage( + id = MessageId.newMutable(), + metadata = mutableMapOf(), + body = mutableMapOf() + ) + fun newSoftMutable() = ParsedMessage( + metadata = mutableMapOf(), + body = mutableMapOf() + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt new file mode 100644 index 000000000..2d65a6dea --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled + +data class RawMessage( + override var id: MessageId = MessageId.DEFAULT_INSTANCE, + override var eventId: EventId? = null, + override var metadata: MutableMap = Message.DEFAULT_METADATA, + override var protocol: String = "", + /** The body is not mutable by default */ + override var body: ByteBuf = Unpooled.EMPTY_BUFFER, +) : Message { + override fun clean() { + check(id !== MessageId.DEFAULT_INSTANCE) { + "Object can be cleaned because 'id' is default instance" + } + check(metadata !== Message.DEFAULT_METADATA) { + "Object can be cleaned because 'metadata' is immutable" + } + check(body !== Unpooled.EMPTY_BUFFER) { + "Object can be cleaned because 'body' is immutable" + } + + id.clean() + eventId = null + metadata.clear() + protocol = "" + body.clear() + } + + override fun softClean() { + check(metadata !== Message.DEFAULT_METADATA) { + "Object can be cleaned because 'metadata' is immutable" + } + + id = MessageId.DEFAULT_INSTANCE + eventId = null + metadata.clear() + protocol = "" + body = Unpooled.EMPTY_BUFFER + } + + companion object { + fun newMutable() = RawMessage( + id = MessageId.newMutable(), + metadata = mutableMapOf(), + body = Unpooled.buffer() + ) + fun newSoftMutable() = RawMessage( + metadata = mutableMapOf() + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouter.kt similarity index 65% rename from src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt rename to src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouter.kt index 3bc43d152..6d4d62833 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouter.kt @@ -13,10 +13,8 @@ * limitations under the License. */ -package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractFilterStrategy -import com.exactpro.th2.common.schema.filter.strategy.impl.AnyMessageFilterStrategy import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.QueueAttribute @@ -24,29 +22,21 @@ import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName -import com.google.protobuf.Message import org.jetbrains.annotations.NotNull -class DemoMessageBatchRouter : AbstractRabbitRouter() { - override fun getDefaultFilterStrategy(): AbstractFilterStrategy { - return AnyMessageFilterStrategy() - } - +class TransportGroupBatchRouter : AbstractRabbitRouter() { override fun splitAndFilter( - message: DemoGroupBatch, + message: GroupBatch, pinConfiguration: @NotNull QueueConfiguration, pinName: PinName, - ): DemoGroupBatch { - //TODO: Implement - whole batch or null - return message - } + ): GroupBatch? = pinConfiguration.filters.filter(message) override fun createSender( pinConfig: QueueConfiguration, pinName: PinName, bookName: BookName, - ): MessageSender { - return DemoMessageBatchSender( + ): MessageSender { + return TransportGroupBatchSender( connectionManager, pinConfig.exchange, pinConfig.routingKey, @@ -58,23 +48,24 @@ class DemoMessageBatchRouter : AbstractRabbitRouter() { override fun createSubscriber( pinConfig: QueueConfiguration, pinName: PinName, - ): MessageSubscriber { - return DemoMessageBatchSubscriber( + ): MessageSubscriber { + return TransportGroupBatchSubscriber( connectionManager, pinConfig.queue, - pinName + pinName, + pinConfig.filters ) } - override fun DemoGroupBatch.toErrorString(): String = toString() + override fun GroupBatch.toErrorString(): String = toString() override fun getRequiredSendAttributes(): Set = REQUIRED_SEND_ATTRIBUTES override fun getRequiredSubscribeAttributes(): Set = REQUIRED_SUBSCRIBE_ATTRIBUTES companion object { - internal const val DEMO_RAW_MESSAGE_TYPE = "DEMO_RAW_MESSAGE" -// internal const val DEMO_RAW_ATTRIBUTE = "demo_raw" // It should be added in future - private val REQUIRED_SUBSCRIBE_ATTRIBUTES = setOf(QueueAttribute.SUBSCRIBE.value) - private val REQUIRED_SEND_ATTRIBUTES = setOf(QueueAttribute.PUBLISH.value) + const val TRANSPORT_GROUP_TYPE = "TRANSPORT_GROUP" + const val TRANSPORT_GROUP_ATTRIBUTE = "transport-group" + private val REQUIRED_SUBSCRIBE_ATTRIBUTES = setOf(QueueAttribute.SUBSCRIBE.value, TRANSPORT_GROUP_ATTRIBUTE) + private val REQUIRED_SEND_ATTRIBUTES = setOf(QueueAttribute.PUBLISH.value, TRANSPORT_GROUP_ATTRIBUTE) } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSender.kt similarity index 65% rename from src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt rename to src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSender.kt index 48feb27d4..40e47fbef 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSender.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSender.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.exactpro.th2.common.metrics.BOOK_NAME_LABEL import com.exactpro.th2.common.metrics.SESSION_GROUP_LABEL @@ -22,42 +22,42 @@ import com.exactpro.th2.common.metrics.TH2_PIN_LABEL import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSender import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter.Companion.DEMO_RAW_MESSAGE_TYPE +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportGroupBatchRouter.Companion.TRANSPORT_GROUP_TYPE import io.prometheus.client.Counter -class DemoMessageBatchSender( +class TransportGroupBatchSender( connectionManager: ConnectionManager, exchangeName: String, routingKey: String, th2Pin: String, bookName: BookName, -) : AbstractRabbitSender( +) : AbstractRabbitSender( connectionManager, exchangeName, routingKey, th2Pin, - DEMO_RAW_MESSAGE_TYPE, + TRANSPORT_GROUP_TYPE, bookName ) { - override fun send(value: DemoGroupBatch) { - DEMO_RAW_MESSAGE_PUBLISH_TOTAL + override fun send(value: GroupBatch) { + TRANSPORT_GROUP_PUBLISH_TOTAL .labels(th2Pin, value.book, value.sessionGroup) .inc(value.groups.size.toDouble()) super.send(value) } - override fun valueToBytes(value: DemoGroupBatch): ByteArray = value.toByteArray() + override fun valueToBytes(value: GroupBatch): ByteArray = value.toByteArray() - override fun toShortTraceString(value: DemoGroupBatch): String = value.toString() + override fun toShortTraceString(value: GroupBatch): String = value.toString() - override fun toShortDebugString(value: DemoGroupBatch): String = value.toString() + override fun toShortDebugString(value: GroupBatch): String = value.toString() companion object { - private val DEMO_RAW_MESSAGE_PUBLISH_TOTAL = Counter.build() - .name("th2_demo_raw_message_publish_total") + private val TRANSPORT_GROUP_PUBLISH_TOTAL = Counter.build() + .name("th2_transport_group_publish_total") .labelNames(TH2_PIN_LABEL, BOOK_NAME_LABEL, SESSION_GROUP_LABEL) - .help("Quantity of published demo raw messages") + .help("Quantity of published transport groups") .register() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSubscriber.kt similarity index 62% rename from src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt rename to src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSubscriber.kt index 0963a7a05..2c996a986 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/DemoMessageBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSubscriber.kt @@ -13,54 +13,53 @@ * limitations under the License. */ -package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.exactpro.th2.common.metrics.BOOK_NAME_LABEL import com.exactpro.th2.common.metrics.SESSION_GROUP_LABEL import com.exactpro.th2.common.metrics.TH2_PIN_LABEL import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback +import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager -import com.exactpro.th2.common.schema.message.impl.rabbitmq.demo.DemoMessageBatchRouter.Companion.DEMO_RAW_MESSAGE_TYPE +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportGroupBatchRouter.Companion.TRANSPORT_GROUP_TYPE import com.rabbitmq.client.Delivery import io.netty.buffer.Unpooled import io.prometheus.client.Counter -class DemoMessageBatchSubscriber( +class TransportGroupBatchSubscriber( connectionManager: ConnectionManager, queue: String, th2Pin: String, -) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, DEMO_RAW_MESSAGE_TYPE) { + private val filters: List, +) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, TRANSPORT_GROUP_TYPE) { - override fun valueFromBytes(body: ByteArray): DemoGroupBatch = Unpooled.wrappedBuffer(body).run(GroupBatchCodec::decode) + override fun valueFromBytes(body: ByteArray): GroupBatch = Unpooled.wrappedBuffer(body).run(GroupBatchCodec::decode) - override fun toShortTraceString(value: DemoGroupBatch): String = value.toString() + override fun toShortTraceString(value: GroupBatch): String = value.toString() - override fun toShortDebugString(value: DemoGroupBatch): String = value.toString() + override fun toShortDebugString(value: GroupBatch): String = value.toString() - override fun filter(batch: DemoGroupBatch): DemoGroupBatch { - //TODO: Implement - whole batch or null - return batch - } + override fun filter(batch: GroupBatch): GroupBatch? = filters.filter(batch) override fun handle( deliveryMetadata: DeliveryMetadata, delivery: Delivery, - value: DemoGroupBatch, + value: GroupBatch, confirmation: ManualAckDeliveryCallback.Confirmation, ) { - DEMO_RAW_MESSAGE_SUBSCRIBE_TOTAL + TRANSPORT_GROUP_SUBSCRIBE_TOTAL .labels(th2Pin, value.book, value.sessionGroup) .inc(value.groups.size.toDouble()) super.handle(deliveryMetadata, delivery, value, confirmation) } companion object { - private val DEMO_RAW_MESSAGE_SUBSCRIBE_TOTAL = Counter.build() - .name("th2_demo_raw_message_subscribe_total") + private val TRANSPORT_GROUP_SUBSCRIBE_TOTAL = Counter.build() + .name("th2_transport_group_subscribe_total") .labelNames(TH2_PIN_LABEL, BOOK_NAME_LABEL, SESSION_GROUP_LABEL) - .help("Quantity of received demo raw messages") + .help("Quantity of received transport groups") .register() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt new file mode 100644 index 000000000..0d5a8e63b --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt @@ -0,0 +1,74 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.* +import com.exactpro.th2.common.schema.filter.strategy.impl.checkFieldValue +import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration +import com.exactpro.th2.common.schema.message.configuration.RouterFilter +import io.netty.buffer.Unpooled + +private typealias ProtoDirection = com.exactpro.th2.common.grpc.Direction + +fun GroupBatch.toByteArray() = Unpooled.buffer().run { + GroupBatchCodec.encode(this@toByteArray, this@run) + ByteArray(readableBytes()).apply(::readBytes) +} + +val Direction.proto: ProtoDirection + get() = when (this) { + Direction.INCOMING -> ProtoDirection.FIRST + Direction.OUTGOING -> ProtoDirection.SECOND + } + +fun Collection.filter(batch: GroupBatch): GroupBatch? { + if (isEmpty()) { + return batch + } + + forEach { filterSet -> + if (!filterSet.metadata[BOOK_KEY].verify(batch.book)) { return@forEach } + if (!filterSet.metadata[SESSION_GROUP_KEY].verify(batch.sessionGroup)) { return@forEach } + + if (!filterSet.metadata[SESSION_ALIAS_KEY].verify(batch.groups) { id.sessionAlias }) { return@forEach } + if (!filterSet.metadata[MESSAGE_TYPE_KEY].verify(batch.groups) { if (this is ParsedMessage) type else "" }) { return@forEach } + if (!filterSet.metadata[DIRECTION_KEY].verify(batch.groups) { id.direction.proto.name }) { return@forEach } + + return batch + } + + return null +} + +private fun Collection?.verify(value: String): Boolean { + if (isNullOrEmpty()) { return true } + return all { it.checkFieldValue(value) } +} + +private inline fun Collection?.verify( + messageGroups: Collection, + value: Message<*>.() -> String +): Boolean { + if (isNullOrEmpty()) { return true } + + // Illegal cases when groups or messages are empty + if (messageGroups.isEmpty()) { return false } + val firstGroup = messageGroups.first() + if (firstGroup.messages.isEmpty()) { return false } + + return all { filter -> filter.checkFieldValue(firstGroup.messages.first().value()) } +} \ No newline at end of file diff --git a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java index 7f731f52c..067d50333 100644 --- a/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java +++ b/src/test/java/com/exactpro/th2/common/event/bean/BaseTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt new file mode 100644 index 000000000..d40d3a218 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt @@ -0,0 +1,173 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import io.netty.buffer.Unpooled +import org.junit.jupiter.api.Test +import java.time.Instant +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals + +class CleanableTest { + + private val filledMessageId = MessageId( + "book", + "sessionGroup", + "sessionAlias", + Direction.OUTGOING, + 1, + mutableListOf(2, 3), + Instant.now(), + ) + private val filledEventId = EventId( + "id", + "book", + "scope", + Instant.now(), + ) + + @Test + fun `event id clean test`() { + val empty = EventId.DEFAULT_INSTANCE + + val mutable = EventId.newMutable().apply { + id = "id" + book = "book" + scope = "scope" + timestamp = Instant.now() + } + assertNotEquals(empty, mutable) + + mutable.clean() + assertEquals(empty, mutable) + } + + @Test + fun `message id clean test`() { + val empty = MessageId.DEFAULT_INSTANCE + + val mutable = MessageId.newMutable().apply { + book = "book" + sessionGroup = "sessionGroup" + sessionAlias = "sessionAlias" + direction = Direction.OUTGOING + sequence = 1 + subsequence = mutableListOf(2, 3) + timestamp = Instant.now() + } + assertNotEquals(empty, mutable) + + mutable.clean() + assertEquals(empty, mutable) + } + + @Test + fun `raw message clean test`() { + val empty = RawMessage() + + val mutable = RawMessage.newMutable().apply { + id = filledMessageId + eventId = filledEventId + metadata["property"] = "value" + protocol = "protocol" + body.writeByte(64) + } + assertNotEquals(empty, mutable) + + mutable.clean() + assertEquals(empty, mutable) + + val softMutable = RawMessage.newSoftMutable().apply { + id = filledMessageId + eventId = filledEventId + metadata["property"] = "value" + protocol = "protocol" + body = Unpooled.buffer().writeByte(64) + } + assertNotEquals(empty, softMutable) + + softMutable.softClean() + assertEquals(empty, softMutable) + } + + @Test + fun `parsed message clean test`() { + val empty = ParsedMessage() + + val mutable = ParsedMessage.newMutable().apply { + id = filledMessageId + eventId = filledEventId + metadata["property"] = "value" + protocol = "protocol" + type = "type" + } + assertNotEquals(empty, mutable) + + mutable.clean() + assertEquals(empty, mutable) + + val softMutable = ParsedMessage.newSoftMutable().apply { + id = filledMessageId + eventId = filledEventId + metadata["property"] = "value" + protocol = "protocol" + type = "type" + } + assertNotEquals(empty, softMutable) + + softMutable.softClean() + assertEquals(empty, softMutable) + } + + @Test + fun `message group clean test`() { + val empty = MessageGroup() + + val mutable = MessageGroup.newMutable().apply { + messages.add(RawMessage()) + } + assertNotEquals(empty, mutable) + + mutable.clean() + assertEquals(empty, mutable) + + mutable.messages.add(ParsedMessage()) + assertNotEquals(empty, mutable) + + mutable.softClean() + assertEquals(empty, mutable) + } + + @Test + fun `group batch clean test`() { + val empty = GroupBatch() + + val mutable = GroupBatch.newMutable().apply { + groups.add(MessageGroup()) + } + assertNotEquals(empty, mutable) + + mutable.clean() + assertEquals(empty, mutable) + + mutable.groups.add(MessageGroup()) + assertNotEquals(empty, mutable) + + mutable.softClean() + assertEquals(empty, mutable) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt new file mode 100644 index 000000000..0f23b6e7b --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import io.netty.buffer.Unpooled +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.time.Instant + +class CodecsTest { + + @Test + fun `decode encode test`() { + val buffer = Unpooled.buffer() + + val message1 = RawMessage( + id = MessageId( + book = "book1", + sessionGroup = "group1", + sessionAlias = "alias1", + direction = Direction.INCOMING, + sequence = 1, + subsequence = mutableListOf(1, 2), + timestamp = Instant.now() + ), + metadata = mutableMapOf( + "prop1" to "value1", + "prop2" to "value2" + ), + protocol = "proto1", + body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3, 4)) + ) + + val message2 = RawMessage( + id = MessageId( + book = "book1", + sessionGroup = "group1", + sessionAlias = "alias2", + direction = Direction.OUTGOING, + sequence = 2, + subsequence = mutableListOf(3, 4), + timestamp = Instant.now() + ), + metadata = mutableMapOf( + "prop3" to "value3", + "prop4" to "value4" + ), + protocol = "proto2", + body = Unpooled.wrappedBuffer(byteArrayOf(5, 6, 7, 8)) + ) + + val message3 = ParsedMessage( + id = MessageId( + book = "book1", + sessionGroup = "group1", + sessionAlias = "alias3", + direction = Direction.OUTGOING, + sequence = 3, + subsequence = mutableListOf(5, 6), + timestamp = Instant.now() + ), + metadata = mutableMapOf( + "prop5" to "value6", + "prop7" to "value8" + ), + protocol = "proto3", + type = "some-type", + body = mutableMapOf( + "simple" to 1, + "list" to listOf(1, 2, 3), + "map" to mapOf("abc" to "cde") + ) + ) + + val batch = GroupBatch( + book = "book1", + sessionGroup = "group1", + groups = mutableListOf(MessageGroup(mutableListOf(message1, message2, message3))) + ) + + GroupBatchCodec.encode(batch, buffer) + val decodedBatch = GroupBatchCodec.decode(buffer) + + assertEquals(batch, decodedBatch) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterIntegrationTest.kt similarity index 96% rename from src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoGroupBatchRouter.kt rename to src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterIntegrationTest.kt index 1cd87b985..e27d3c9a3 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/IntegrationTestDemoGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterIntegrationTest.kt @@ -12,7 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.exactpro.th2.common.annotations.IntegrationTest import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration @@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit import kotlin.test.assertTrue @IntegrationTest -class IntegrationTestDemoGroupBatchRouter { +class TransportGroupBatchRouterIntegrationTest { @Test fun `subscribe to exclusive queue`() { @@ -49,7 +49,7 @@ class IntegrationTestDemoGroupBatchRouter { val counter = CountDownLatch(1) val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } try { - secondRouter.sendExclusive(monitor.queue, DemoGroupBatch()) + secondRouter.sendExclusive(monitor.queue, GroupBatch()) assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } } finally { @@ -80,7 +80,7 @@ class IntegrationTestDemoGroupBatchRouter { val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } try { - secondRouter.sendExclusive(monitor.queue, DemoGroupBatch()) + secondRouter.sendExclusive(monitor.queue, GroupBatch()) assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } } finally { @@ -93,7 +93,7 @@ class IntegrationTestDemoGroupBatchRouter { } } - private fun createRouter(connectionManager: ConnectionManager) = DemoMessageBatchRouter() + private fun createRouter(connectionManager: ConnectionManager) = TransportGroupBatchRouter() .apply { init( DefaultMessageRouterContext( diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt similarity index 62% rename from src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt rename to src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt index 82f922f53..976b9e94b 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/demo/TestDemoGroupBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt @@ -14,35 +14,26 @@ * limitations under the License. */ -package com.exactpro.th2.common.schema.message.impl.rabbitmq.demo +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import com.exactpro.th2.common.event.bean.BaseTest.BOOK_NAME -import com.exactpro.th2.common.event.bean.BaseTest.BOX_CONFIGURATION -import com.exactpro.th2.common.event.bean.BaseTest.SESSION_ALIAS -import com.exactpro.th2.common.event.bean.BaseTest.SESSION_GROUP +import com.exactpro.th2.common.event.bean.BaseTest.* import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.schema.message.ExclusiveSubscriberMonitor import com.exactpro.th2.common.schema.message.MessageRouter -import com.exactpro.th2.common.schema.message.configuration.GlobalNotificationConfiguration -import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration -import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration +import com.exactpro.th2.common.schema.message.configuration.* import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportGroupBatchRouter.Companion.TRANSPORT_GROUP_ATTRIBUTE import io.netty.buffer.Unpooled import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.function.Executable -import org.mockito.kotlin.any -import org.mockito.kotlin.anyOrNull -import org.mockito.kotlin.argumentCaptor -import org.mockito.kotlin.eq -import org.mockito.kotlin.mock -import org.mockito.kotlin.verify +import org.mockito.kotlin.* -class TestDemoGroupBatchRouter { +class TransportGroupBatchRouterTest { private val connectionConfiguration = ConnectionManagerConfiguration() private val monitor: ExclusiveSubscriberMonitor = mock { } private val connectionManager: ConnectionManager = mock { @@ -58,41 +49,70 @@ class TestDemoGroupBatchRouter { routingKey = "", queue = "subscribe", exchange = "test-exchange", - attributes = listOf("subscribe") + attributes = listOf("subscribe", TRANSPORT_GROUP_ATTRIBUTE) ), "test-pin1" to QueueConfiguration( routingKey = "test", queue = "", exchange = "test-exchange", - attributes = listOf("publish"), + attributes = listOf("publish", TRANSPORT_GROUP_ATTRIBUTE), + filters = listOf( + MqRouterFilterConfiguration( + metadata = listOf( + FieldFilterConfiguration( + fieldName = "message_type", + expectedValue = "test-message", + operation = FieldFilterOperation.EQUAL + ) + ) + ) + ) ), "test-pin2" to QueueConfiguration( routingKey = "test2", queue = "", exchange = "test-exchange", - attributes = listOf("publish", "test"), - ) - ) - ) - - @Test - fun `publishes to the correct pin according to attributes`() { - val batch = DemoGroupBatch( - BOOK_NAME, - SESSION_GROUP, - mutableListOf( - DemoMessageGroup( - mutableListOf( - DemoRawMessage( - DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), - body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3)) + attributes = listOf("publish", TRANSPORT_GROUP_ATTRIBUTE, "test"), + filters = listOf( + MqRouterFilterConfiguration( + metadata = listOf( + FieldFilterConfiguration( + fieldName = "message_type", + expectedValue = "test-message", + operation = FieldFilterOperation.EQUAL + ) ) ) ) ) ) + ) + + @Test + fun `publishes message group batch with metadata`() { + val batch = createGroupBatch("test-message") + router.send(batch, "test") + val captor = argumentCaptor() + verify(connectionManager).basicPublish(eq("test-exchange"), eq("test2"), anyOrNull(), captor.capture()) + val publishedBytes = captor.firstValue + assertArrayEquals(batch.toByteArray(), publishedBytes) { + "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" + } + } + + @Test + fun `does not publish anything if all messages are filtered`() { + router.send(createGroupBatch("test-message1")) + + verify(connectionManager, never()).basicPublish(any(), any(), anyOrNull(), any()) + } + + @Test + fun `publishes to the correct pin according to attributes`() { + val batch = createGroupBatch("test-message") + router.send(batch, "test") val captor = argumentCaptor() verify(connectionManager).basicPublish(eq("test-exchange"), eq("test2"), anyOrNull(), captor.capture()) @@ -105,10 +125,10 @@ class TestDemoGroupBatchRouter { @Test fun `reports about extra pins matches the publication`() { Assertions.assertThrows(IllegalStateException::class.java) { - router.send(DEMO_MESSAGE_BATCH) + router.send(createGroupBatch("test-message")) }.apply { Assertions.assertEquals( - "Found incorrect number of pins [test-pin1, test-pin2] to the send operation by attributes [publish] and filters, expected 1, actual 2", + "Found incorrect number of pins [test-pin1, test-pin2] to the send operation by attributes [publish, $TRANSPORT_GROUP_ATTRIBUTE] and filters, expected 1, actual 2", message ) } @@ -117,10 +137,10 @@ class TestDemoGroupBatchRouter { @Test fun `reports about no pins matches the publication`() { Assertions.assertThrows(IllegalStateException::class.java) { - router.send(DEMO_MESSAGE_BATCH, "unexpected") + router.send(TRANSPORT_BATCH, "unexpected") }.apply { Assertions.assertEquals( - "Found incorrect number of pins [] to the send operation by attributes [unexpected, publish] and filters, expected 1, actual 0", + "Found incorrect number of pins [] to the send operation by attributes [unexpected, publish, $TRANSPORT_GROUP_ATTRIBUTE] and filters, expected 1, actual 0", message ) } @@ -128,12 +148,13 @@ class TestDemoGroupBatchRouter { @Test fun `publishes to all correct pin according to attributes`() { - router.sendAll(DEMO_MESSAGE_BATCH) + val batch = createGroupBatch("test-message") + router.sendAll(batch) val captor = argumentCaptor() verify(connectionManager).basicPublish(eq("test-exchange"), eq("test"), anyOrNull(), captor.capture()) verify(connectionManager).basicPublish(eq("test-exchange"), eq("test2"), anyOrNull(), captor.capture()) - val originalBytes = DEMO_MESSAGE_BATCH.toByteArray() + val originalBytes = batch.toByteArray() Assertions.assertAll( Executable { val publishedBytes = captor.firstValue @@ -159,23 +180,36 @@ class TestDemoGroupBatchRouter { routingKey = "publish", queue = "", exchange = "test-exchange", - attributes = listOf("publish", "test") + attributes = listOf("publish", TRANSPORT_GROUP_ATTRIBUTE, "test") ), "test1" to QueueConfiguration( routingKey = "", queue = "queue1", exchange = "test-exchange", - attributes = listOf("subscribe", "1") + attributes = listOf("subscribe", TRANSPORT_GROUP_ATTRIBUTE, "1") ), "test2" to QueueConfiguration( routingKey = "", queue = "queue2", exchange = "test-exchange", - attributes = listOf("subscribe", "2") + attributes = listOf("subscribe", TRANSPORT_GROUP_ATTRIBUTE, "2") ) ) ) + @Test + fun `publishes message group batch with metadata`() { + val batch = createGroupBatch("test-message") + router.send(batch, "test") + + val captor = argumentCaptor() + verify(connectionManager).basicPublish(eq("test-exchange"), eq("publish"), anyOrNull(), captor.capture()) + val publishedBytes = captor.firstValue + assertArrayEquals(batch.toByteArray(), publishedBytes) { + "Unexpected batch published: ${MessageGroupBatch.parseFrom(publishedBytes)}" + } + } + @Test fun `subscribes to correct queue`() { val monitor = router.subscribe(mock { }, "1") @@ -183,6 +217,7 @@ class TestDemoGroupBatchRouter { verify(connectionManager).basicConsume(eq("queue1"), any(), any()) } + @Test fun `subscribes with manual ack to correct queue`() { val monitor = router.subscribeWithManualAck(mock { }, "1") @@ -205,7 +240,7 @@ class TestDemoGroupBatchRouter { Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }) } .apply { Assertions.assertEquals( - "Found incorrect number of pins [test1, test2] to subscribe operation by attributes [subscribe] and filters, expected 1, actual 2", + "Found incorrect number of pins [test1, test2] to subscribe operation by attributes [subscribe, $TRANSPORT_GROUP_ATTRIBUTE] and filters, expected 1, actual 2", message ) } @@ -215,19 +250,29 @@ class TestDemoGroupBatchRouter { fun `reports if no queue matches`() { Assertions.assertAll( Executable { - Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }, "unexpected") } + Assertions.assertThrows(IllegalStateException::class.java) { + router.subscribe( + mock { }, + "unexpected" + ) + } .apply { Assertions.assertEquals( - "Found incorrect number of pins [] to subscribe operation by attributes [unexpected, subscribe] and filters, expected 1, actual 0", + "Found incorrect number of pins [] to subscribe operation by attributes [unexpected, subscribe, $TRANSPORT_GROUP_ATTRIBUTE] and filters, expected 1, actual 0", message ) } }, Executable { - Assertions.assertThrows(IllegalStateException::class.java) { router.subscribeAll(mock { }, "unexpected") } + Assertions.assertThrows(IllegalStateException::class.java) { + router.subscribeAll( + mock { }, + "unexpected" + ) + } .apply { Assertions.assertEquals( - "Found incorrect number of pins [] to subscribe all operation by attributes [unexpected, subscribe] and filters, expected 1 or more, actual 0", + "Found incorrect number of pins [] to subscribe all operation by attributes [unexpected, subscribe, $TRANSPORT_GROUP_ATTRIBUTE] and filters, expected 1 or more, actual 0", message ) } @@ -236,8 +281,23 @@ class TestDemoGroupBatchRouter { } } - private fun createRouter(pins: Map): MessageRouter = - DemoMessageBatchRouter().apply { + private fun createGroupBatch(messageType: String) = GroupBatch( + BOOK_NAME, + SESSION_GROUP, + mutableListOf( + MessageGroup( + mutableListOf( + ParsedMessage( + MessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), + type = messageType + ) + ) + ) + ) + ) + + private fun createRouter(pins: Map): MessageRouter = + TransportGroupBatchRouter().apply { init( DefaultMessageRouterContext( connectionManager, @@ -249,14 +309,14 @@ class TestDemoGroupBatchRouter { } companion object { - private val DEMO_MESSAGE_BATCH = DemoGroupBatch( + private val TRANSPORT_BATCH = GroupBatch( BOOK_NAME, SESSION_GROUP, mutableListOf( - DemoMessageGroup( + MessageGroup( mutableListOf( - DemoRawMessage( - DemoMessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), + RawMessage( + MessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3)) ) ) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt new file mode 100644 index 000000000..4fb81b696 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt @@ -0,0 +1,116 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.* +import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration +import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation.* +import com.exactpro.th2.common.schema.message.configuration.MqRouterFilterConfiguration +import com.exactpro.th2.common.schema.message.configuration.RouterFilter +import com.exactpro.th2.common.util.emptyMultiMap +import org.apache.commons.collections4.MultiMapUtils +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertSame + +class TransportUtilsTest { + + private val bookA = "bookA" + private val bookB = "bookB" + + private val groupA = "groupA" + private val groupB = "groupB" + + private val msgType = "msg-type" + + private val directionA = "SECOND" + private val directionB = "FIRST" + + private val routerFilters = listOf( + MqRouterFilterConfiguration( + MultiMapUtils.newListValuedHashMap().apply { + putAll(BOOK_KEY, listOf( + FieldFilterConfiguration(BOOK_KEY, bookA, EQUAL), + FieldFilterConfiguration(BOOK_KEY, "*A", WILDCARD), + FieldFilterConfiguration(BOOK_KEY, null, NOT_EMPTY), + FieldFilterConfiguration(BOOK_KEY, bookB, NOT_EQUAL) + )) + put(SESSION_GROUP_KEY, FieldFilterConfiguration(SESSION_GROUP_KEY, "*A", WILDCARD)) + put(SESSION_ALIAS_KEY, FieldFilterConfiguration(SESSION_ALIAS_KEY, null, EMPTY)) + put(MESSAGE_TYPE_KEY, FieldFilterConfiguration(MESSAGE_TYPE_KEY, null, NOT_EMPTY)) + put(DIRECTION_KEY, FieldFilterConfiguration(DIRECTION_KEY, directionB, NOT_EQUAL)) + }, + emptyMultiMap() + ), + MqRouterFilterConfiguration( + MultiMapUtils.newListValuedHashMap().apply { + putAll(BOOK_KEY, listOf( + FieldFilterConfiguration(BOOK_KEY, bookB, EQUAL), + FieldFilterConfiguration(BOOK_KEY, "*B", WILDCARD), + FieldFilterConfiguration(BOOK_KEY, null, NOT_EMPTY), + FieldFilterConfiguration(BOOK_KEY, bookA, NOT_EQUAL) + )) + put(SESSION_GROUP_KEY, FieldFilterConfiguration(SESSION_GROUP_KEY, "*B", WILDCARD)) + put(SESSION_ALIAS_KEY, FieldFilterConfiguration(SESSION_ALIAS_KEY, null, EMPTY)) + put(MESSAGE_TYPE_KEY, FieldFilterConfiguration(MESSAGE_TYPE_KEY, null, NOT_EMPTY)) + put(DIRECTION_KEY, FieldFilterConfiguration(DIRECTION_KEY, directionA, NOT_EQUAL)) + }, + emptyMultiMap() + ) + ) + + @Test + fun `empty filter test`() { + val group = GroupBatch() + assertSame(group, listOf().filter(group)) + + group.book = bookB + group.sessionGroup = groupA + assertSame(group, listOf().filter(group)) + + } + + @Test + fun `filter test`() { + val batch = GroupBatch.newMutable() + assertNull(routerFilters.filter(batch)) + + batch.book = bookA + assertNull(routerFilters.filter(batch)) + batch.sessionGroup = groupA + assertNull(routerFilters.filter(batch)) + val group = MessageGroup.newMutable() + batch.groups.add(group) + assertNull(routerFilters.filter(batch)) + + val parsedMessage = ParsedMessage.newMutable() + group.messages.add(parsedMessage) + assertNull(routerFilters.filter(batch)) + + parsedMessage.type = msgType + assertNull(routerFilters.filter(batch)) + + parsedMessage.id = MessageId(direction = Direction.OUTGOING) + assertEquals(batch, routerFilters.filter(batch)) + + parsedMessage.id = MessageId(direction = Direction.INCOMING) + batch.sessionGroup = groupB + batch.book = bookB + assertEquals(batch, routerFilters.filter(batch)) + } +} \ No newline at end of file From 78df5e80551c2ee717b7171f2a81cb171dc47333 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 18 Apr 2023 16:38:30 +0400 Subject: [PATCH 118/154] Removed book and group from message id --- build.gradle | 2 +- .../schema/factory/AbstractCommonFactory.java | 9 ++-- .../strategy/impl/AnyMessageFilterStrategy.kt | 6 +-- .../group/RabbitMessageGroupBatchRouter.kt | 6 +-- .../message/impl/rabbitmq/transport/Codecs.kt | 12 +---- .../impl/rabbitmq/transport/EventId.kt | 1 + .../impl/rabbitmq/transport/GroupBatch.kt | 1 + .../impl/rabbitmq/transport/MessageGroup.kt | 1 + .../impl/rabbitmq/transport/MessageId.kt | 5 +- .../impl/rabbitmq/transport/ParsedMessage.kt | 2 + .../impl/rabbitmq/transport/RawMessage.kt | 2 + .../impl/rabbitmq/transport/TransportUtils.kt | 3 +- .../common/schema/TestJsonConfiguration.kt | 48 ++++--------------- .../impl/TestAnyMessageFilterStrategy.kt | 8 ++-- .../impl/rabbitmq/transport/CleanableTest.kt | 4 -- .../impl/rabbitmq/transport/CodecsTest.kt | 6 --- .../TransportGroupBatchRouterTest.kt | 4 +- .../com/exactpro/th2/common/TestUtils.kt | 3 +- 18 files changed, 36 insertions(+), 87 deletions(-) diff --git a/build.gradle b/build.gradle index 5bd3a266d..31334044c 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.0.0-separate-executor-4540242539-6c92e55-SNAPSHOT' + cradleVersion = '5.1.0-separate-executor-4678409529-5d1e60e-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index dd1e34b74..48cf769f7 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -118,7 +118,7 @@ public abstract class AbstractCommonFactory implements AutoCloseable { protected static final String LOG4J_PROPERTIES_DEFAULT_PATH = "/var/th2/config"; protected static final String LOG4J2_PROPERTIES_NAME = "log4j2.properties"; - protected static final ObjectMapper MAPPER = new ObjectMapper(); + public static final ObjectMapper MAPPER = new ObjectMapper(); static { MAPPER.registerModules( @@ -499,15 +499,15 @@ public BoxConfiguration getBoxConfiguration() { return getConfigurationOrLoad(BoxConfiguration.class, true); } - protected CradleConfidentialConfiguration getCradleConfidentialConfiguration() { + private CradleConfidentialConfiguration getCradleConfidentialConfiguration() { return getConfigurationOrLoad(CradleConfidentialConfiguration.class, false); } - protected CradleNonConfidentialConfiguration getCradleNonConfidentialConfiguration() { + private CradleNonConfidentialConfiguration getCradleNonConfidentialConfiguration() { return getConfigurationOrLoad(CradleNonConfidentialConfiguration.class, true); } - protected CassandraStorageSettings getCassandraStorageSettings() { + private CassandraStorageSettings getCassandraStorageSettings() { return getConfigurationOrLoad(CassandraStorageSettings.class, true); } @@ -534,6 +534,7 @@ public CradleManager getCradleManager() { // Deserialize on config by two different beans for backward compatibility CradleNonConfidentialConfiguration nonConfidentialConfiguration = getCradleNonConfidentialConfiguration(); + // FIXME: this approach should be replaced to module structure in future CassandraStorageSettings cassandraStorageSettings = getCassandraStorageSettings(); cassandraStorageSettings.setKeyspace(confidentialConfiguration.getKeyspace()); diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt index 7d6231020..5d76c600e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt @@ -20,7 +20,7 @@ import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.message.toJson import com.google.protobuf.Message -class AnyMessageFilterStrategy : AbstractFilterStrategy() { +object AnyMessageFilterStrategy : AbstractFilterStrategy() { override fun getFields(message: Message): MutableMap { check(message is AnyMessage) { "Message is not an ${AnyMessage::class.qualifiedName}: ${message.toJson()}" } @@ -54,8 +54,4 @@ class AnyMessageFilterStrategy : AbstractFilterStrategy() { return result } - - companion object { - val INSTANCE = AnyMessageFilterStrategy() - } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt index 1cf80c7d8..23c0e6d25 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt @@ -17,7 +17,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.group import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.metrics.* -import com.exactpro.th2.common.schema.filter.strategy.impl.AnyMessageFilterStrategy.Companion.INSTANCE +import com.exactpro.th2.common.schema.filter.strategy.impl.AnyMessageFilterStrategy import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration @@ -44,7 +44,7 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() val builder = message.toBuilderWithMetadata() message.groupsList.forEach { group -> - if (group.messagesList.all { INSTANCE.verify(it, pinConfiguration.filters) }) { + if (group.messagesList.all { AnyMessageFilterStrategy.verify(it, pinConfiguration.filters) }) { builder.addGroups(group) } else { incrementDroppedMetrics( @@ -80,7 +80,7 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() return RabbitMessageGroupBatchSubscriber( connectionManager, pinConfig.queue, - { msg: Message, filters: List -> INSTANCE.verify(msg, filters) }, + { msg: Message, filters: List -> AnyMessageFilterStrategy.verify(msg, filters) }, pinName, pinConfig.filters, connectionManager.configuration.messageRecursionLimit diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt index 712122f9d..345f9fba2 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt @@ -60,6 +60,8 @@ enum class ValueType(val codec: ValueCodec<*>) { GROUP_LIST(GroupListCodec); companion object { + fun forId(id: UByte): ValueType = MAPPING[id.toInt()] ?: UNKNOWN + private val MAPPING: Array = arrayOfNulls(UByte.MAX_VALUE.toInt()).apply { ValueType.values().forEach { this[it.codec.type.toInt()]?.let { previous -> @@ -68,8 +70,6 @@ enum class ValueType(val codec: ValueCodec<*>) { this[it.codec.type.toInt()] = it } } - - fun forId(id: UByte): ValueType = MAPPING[id.toInt()] ?: UNKNOWN } } @@ -376,14 +376,6 @@ object GroupBatchCodec : AbstractCodec(50u) { else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") } } - - groups.forEach { group -> - group.messages.forEach { message -> - val id = message.id - id.book = book - id.sessionGroup = sessionGroup - } - } } override fun write(buffer: ByteBuf, value: GroupBatch) { diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt index b6afc0606..51a5e2aa7 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt @@ -37,6 +37,7 @@ data class EventId( companion object { val DEFAULT_INSTANCE: EventId = EventId() // FIXME: do smth about its mutability + @JvmStatic fun newMutable() = EventId() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt index 4d1da7479..3c9c06ae0 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt @@ -33,6 +33,7 @@ data class GroupBatch( companion object { val DEFAULT_GROUPS: MutableList = Collections.emptyList() + @JvmStatic fun newMutable() = GroupBatch( groups = mutableListOf() ) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt index 2ba360c84..7d096cf91 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt @@ -30,6 +30,7 @@ data class MessageGroup( companion object { val DEFAULT_MESSAGES: MutableList> = Collections.emptyList() + @JvmStatic fun newMutable() = MessageGroup(mutableListOf()) } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt index 272e9e595..2dc61d2d7 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt @@ -20,8 +20,6 @@ import java.time.Instant import java.util.* data class MessageId( - var book: String = "", - var sessionGroup: String = "", var sessionAlias: String = "", var direction: Direction = Direction.INCOMING, var sequence: Long = 0, @@ -37,8 +35,6 @@ data class MessageId( "Object can be cleaned because 'subsequence' is immutable" } - book = "" - sessionGroup = "" sessionAlias = "" direction = Direction.INCOMING sequence = 0 @@ -49,6 +45,7 @@ data class MessageId( companion object { val DEFAULT_SUBSEQUENCE: MutableList = Collections.emptyList() val DEFAULT_INSTANCE: MessageId = MessageId() // FIXME: do smth about its mutability + @JvmStatic fun newMutable() = MessageId( subsequence = mutableListOf() ) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt index 5e58601ee..a5d5d5cfd 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt @@ -64,11 +64,13 @@ data class ParsedMessage( companion object { val DEFAULT_BODY: MutableMap = Collections.emptyMap() + @JvmStatic fun newMutable() = ParsedMessage( id = MessageId.newMutable(), metadata = mutableMapOf(), body = mutableMapOf() ) + @JvmStatic fun newSoftMutable() = ParsedMessage( metadata = mutableMapOf(), body = mutableMapOf() diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt index 2d65a6dea..94166acfd 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt @@ -58,11 +58,13 @@ data class RawMessage( } companion object { + @JvmStatic fun newMutable() = RawMessage( id = MessageId.newMutable(), metadata = mutableMapOf(), body = Unpooled.buffer() ) + @JvmStatic fun newSoftMutable() = RawMessage( metadata = mutableMapOf() ) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt index 0d5a8e63b..81944a0dd 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt @@ -21,8 +21,7 @@ import com.exactpro.th2.common.schema.filter.strategy.impl.checkFieldValue import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter import io.netty.buffer.Unpooled - -private typealias ProtoDirection = com.exactpro.th2.common.grpc.Direction +import com.exactpro.th2.common.grpc.Direction as ProtoDirection fun GroupBatch.toByteArray() = Unpooled.buffer().run { GroupBatchCodec.encode(this@toByteArray, this@run) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt index 1d00385a5..cfa9efb5d 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt @@ -19,25 +19,12 @@ import com.exactpro.cradle.cassandra.CassandraStorageSettings import com.exactpro.th2.common.metrics.PrometheusConfiguration import com.exactpro.th2.common.schema.cradle.CradleConfidentialConfiguration import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration -import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration -import com.exactpro.th2.common.schema.grpc.configuration.GrpcEndpointConfiguration -import com.exactpro.th2.common.schema.grpc.configuration.GrpcRawRobinStrategy -import com.exactpro.th2.common.schema.grpc.configuration.GrpcServerConfiguration -import com.exactpro.th2.common.schema.grpc.configuration.GrpcServiceConfiguration -import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration -import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation -import com.exactpro.th2.common.schema.message.configuration.GlobalNotificationConfiguration -import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration -import com.exactpro.th2.common.schema.message.configuration.MqRouterFilterConfiguration -import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration +import com.exactpro.th2.common.schema.factory.AbstractCommonFactory.MAPPER +import com.exactpro.th2.common.schema.grpc.configuration.* +import com.exactpro.th2.common.schema.message.configuration.* import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration import com.exactpro.th2.common.schema.strategy.route.impl.RobinRoutingStrategy -import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.kotlin.KotlinFeature -import com.fasterxml.jackson.module.kotlin.KotlinModule import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -103,7 +90,7 @@ class TestJsonConfiguration { @Test fun `test cassandra storage settings json configuration deserialize`() { - assertTrue(EqualsBuilder.reflectionEquals(OBJECT_MAPPER.readValue(CASSANDRA_STORAGE_SETTINGS_JSON, CASSANDRA_STORAGE_SETTINGS::class.java), CASSANDRA_STORAGE_SETTINGS)) + assertTrue(EqualsBuilder.reflectionEquals(MAPPER.readValue(CASSANDRA_STORAGE_SETTINGS_JSON, CASSANDRA_STORAGE_SETTINGS::class.java), CASSANDRA_STORAGE_SETTINGS)) } @Test @@ -113,8 +100,8 @@ class TestJsonConfiguration { @Test fun `test json configuration deserialize combo`() { - assertTrue(EqualsBuilder.reflectionEquals(CASSANDRA_STORAGE_SETTINGS, OBJECT_MAPPER.readValue(CRADLE_NON_CONFIDENTIAL_COMBO_CONF_JSON, CASSANDRA_STORAGE_SETTINGS::class.java))) - assertEquals(CRADLE_NON_CONFIDENTIAL_CONF, OBJECT_MAPPER.readValue(CRADLE_NON_CONFIDENTIAL_COMBO_CONF_JSON, CRADLE_NON_CONFIDENTIAL_CONF::class.java)) + assertTrue(EqualsBuilder.reflectionEquals(CASSANDRA_STORAGE_SETTINGS, MAPPER.readValue(CRADLE_NON_CONFIDENTIAL_COMBO_CONF_JSON, CASSANDRA_STORAGE_SETTINGS::class.java))) + assertEquals(CRADLE_NON_CONFIDENTIAL_CONF, MAPPER.readValue(CRADLE_NON_CONFIDENTIAL_COMBO_CONF_JSON, CRADLE_NON_CONFIDENTIAL_CONF::class.java)) } @Test @@ -128,20 +115,16 @@ class TestJsonConfiguration { } private fun testSerializeAndDeserialize(configuration: Any) { - OBJECT_MAPPER.writeValueAsString(configuration).also { jsonString -> + MAPPER.writeValueAsString(configuration).also { jsonString -> testDeserialize(jsonString, configuration) } } private fun testDeserialize(json: String, obj: Any) { - assertEquals(obj, OBJECT_MAPPER.readValue(json, obj::class.java)) + assertEquals(obj, MAPPER.readValue(json, obj::class.java)) } companion object { - @JvmStatic - private val OBJECT_MAPPER: ObjectMapper = ObjectMapper() - .registerModule(JavaTimeModule()) - @JvmStatic private val CONF_DIR = Path.of("test_json_configurations") @@ -271,21 +254,6 @@ class TestJsonConfiguration { private val PROMETHEUS_CONF_JSON = loadConfJson("prometheus") private val PROMETHEUS_CONF = PrometheusConfiguration("123.3.3.3", 1234, false) - init { - OBJECT_MAPPER.registerModule( - KotlinModule.Builder() - .withReflectionCacheSize(512) - .configure(KotlinFeature.NullToEmptyCollection, false) - .configure(KotlinFeature.NullToEmptyMap, false) - .configure(KotlinFeature.NullIsSameAsDefault, false) - .configure(KotlinFeature.SingletonSupport, false) - .configure(KotlinFeature.StrictNullChecks, false) - .build() - ) - - OBJECT_MAPPER.registerModule(RoutingStrategyModule(OBJECT_MAPPER)) - } - private fun loadConfJson(fileName: String): String { val path = CONF_DIR.resolve(fileName) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt index c1e7ada7e..bd59f34bc 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt @@ -33,12 +33,10 @@ import org.junit.jupiter.params.provider.Arguments.arguments import org.junit.jupiter.params.provider.MethodSource class TestAnyMessageFilterStrategy { - private val strategy = AnyMessageFilterStrategy() - @ParameterizedTest @MethodSource("parsedMessages") fun `matches the parsed message by message type with single filter`(anyMessage: AnyMessage, expectMatch: Boolean) { - val match = strategy.verify( + val match = AnyMessageFilterStrategy.verify( anyMessage, MqRouterFilterConfiguration( metadata = MultiMapUtils.newListValuedHashMap().apply { @@ -56,7 +54,7 @@ class TestAnyMessageFilterStrategy { @ParameterizedTest @MethodSource("messages") fun `matches the parsed message by direction with single filter`(message: AnyMessage, expectMatch: Boolean) { - val match = strategy.verify( + val match = AnyMessageFilterStrategy.verify( message, MqRouterFilterConfiguration( metadata = MultiMapUtils.newListValuedHashMap().apply { @@ -74,7 +72,7 @@ class TestAnyMessageFilterStrategy { @ParameterizedTest @MethodSource("messages") fun `matches the parsed message by alias with single filter`(message: AnyMessage, expectMatch: Boolean) { - val match = strategy.verify( + val match = AnyMessageFilterStrategy.verify( message, MqRouterFilterConfiguration( metadata = MultiMapUtils.newListValuedHashMap().apply { diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt index d40d3a218..b6498baf5 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt @@ -25,8 +25,6 @@ import kotlin.test.assertNotEquals class CleanableTest { private val filledMessageId = MessageId( - "book", - "sessionGroup", "sessionAlias", Direction.OUTGOING, 1, @@ -61,8 +59,6 @@ class CleanableTest { val empty = MessageId.DEFAULT_INSTANCE val mutable = MessageId.newMutable().apply { - book = "book" - sessionGroup = "sessionGroup" sessionAlias = "sessionAlias" direction = Direction.OUTGOING sequence = 1 diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt index 0f23b6e7b..5ce8f6061 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt @@ -29,8 +29,6 @@ class CodecsTest { val message1 = RawMessage( id = MessageId( - book = "book1", - sessionGroup = "group1", sessionAlias = "alias1", direction = Direction.INCOMING, sequence = 1, @@ -47,8 +45,6 @@ class CodecsTest { val message2 = RawMessage( id = MessageId( - book = "book1", - sessionGroup = "group1", sessionAlias = "alias2", direction = Direction.OUTGOING, sequence = 2, @@ -65,8 +61,6 @@ class CodecsTest { val message3 = ParsedMessage( id = MessageId( - book = "book1", - sessionGroup = "group1", sessionAlias = "alias3", direction = Direction.OUTGOING, sequence = 3, diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt index 976b9e94b..1787f19a4 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt @@ -288,7 +288,7 @@ class TransportGroupBatchRouterTest { MessageGroup( mutableListOf( ParsedMessage( - MessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), + MessageId(SESSION_ALIAS), type = messageType ) ) @@ -316,7 +316,7 @@ class TransportGroupBatchRouterTest { MessageGroup( mutableListOf( RawMessage( - MessageId(BOOK_NAME, SESSION_GROUP, SESSION_ALIAS), + MessageId(SESSION_ALIAS), body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3)) ) ) diff --git a/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt b/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt index f73e4813d..52605159b 100644 --- a/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt +++ b/src/testFixtures/kotlin/com/exactpro/th2/common/TestUtils.kt @@ -175,6 +175,7 @@ fun Message.assertDouble(name: String, expected: Double? = null): Double { return actual } +@Suppress("UNCHECKED_CAST") fun Message.assertValue(name: String, expected: T? = null): T { this.assertContains(name) val actual = when (expected) { @@ -185,7 +186,7 @@ fun Message.assertValue(name: String, expected: T? = null): T { is List<*> -> this.getList(name) is String -> this.getString(name) null -> this[name] - else -> error("Cannot assert $name field value. Expected value type is not supported: ${expected!!::class.simpleName}") + else -> error("Cannot assert $name field value. Expected value type is not supported: ${expected.let { it::class.simpleName }}") }!! expected?.let { Assertions.assertEquals(expected, actual) {"Unexpected $name field value"} From 3398cd86f1302d41587871b78688a2f9868a42e9 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 19 Apr 2023 10:58:01 +0400 Subject: [PATCH 119/154] Moved IntegrationTest to Fixtures --- .../kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{test => testFixtures}/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt (100%) diff --git a/src/test/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt b/src/testFixtures/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt similarity index 100% rename from src/test/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt rename to src/testFixtures/kotlin/com/exactpro/th2/common/annotations/IntegrationTest.kt From 21213681630d2bf338059d1dcc76e715860f5673 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 25 Apr 2023 18:05:16 +0400 Subject: [PATCH 120/154] Added utility functions for transport classes * use HashMap instead of LinkedHashMap in transport classes * updated cradle dependency --- build.gradle | 2 +- .../message/impl/rabbitmq/transport/Codecs.kt | 2 +- .../impl/rabbitmq/transport/ParsedMessage.kt | 8 ++--- .../impl/rabbitmq/transport/RawMessage.kt | 4 +-- .../impl/rabbitmq/transport/TransportUtils.kt | 31 ++++++++++++++++++- 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 31334044c..6e9cdf517 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.1.0-separate-executor-4678409529-5d1e60e-SNAPSHOT' + cradleVersion = '5.1.0-separate-executor-4783246997-dda4d35-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt index 345f9fba2..69dc3f610 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt @@ -150,7 +150,7 @@ abstract class MapCodec( private val keyCodec: ValueCodec, private val valueCodec: ValueCodec, ) : AbstractCodec>(type) { - override fun read(buffer: ByteBuf): MutableMap = mutableMapOf().apply { + override fun read(buffer: ByteBuf): MutableMap = hashMapOf().apply { while (buffer.isReadable) { this[keyCodec.decode(buffer)] = valueCodec.decode(buffer) } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt index a5d5d5cfd..5c39dd167 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt @@ -67,13 +67,13 @@ data class ParsedMessage( @JvmStatic fun newMutable() = ParsedMessage( id = MessageId.newMutable(), - metadata = mutableMapOf(), - body = mutableMapOf() + metadata = hashMapOf(), + body = hashMapOf() ) @JvmStatic fun newSoftMutable() = ParsedMessage( - metadata = mutableMapOf(), - body = mutableMapOf() + metadata = hashMapOf(), + body = hashMapOf() ) } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt index 94166acfd..a00b2624c 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt @@ -61,12 +61,12 @@ data class RawMessage( @JvmStatic fun newMutable() = RawMessage( id = MessageId.newMutable(), - metadata = mutableMapOf(), + metadata = hashMapOf(), body = Unpooled.buffer() ) @JvmStatic fun newSoftMutable() = RawMessage( - metadata = mutableMapOf() + metadata = hashMapOf() ) } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt index 81944a0dd..cbf59c5f2 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt @@ -16,18 +16,24 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport +import com.exactpro.th2.common.grpc.EventID +import com.exactpro.th2.common.grpc.MessageID +import com.exactpro.th2.common.message.toTimestamp import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.* import com.exactpro.th2.common.schema.filter.strategy.impl.checkFieldValue import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter +import io.netty.buffer.ByteBuf import io.netty.buffer.Unpooled import com.exactpro.th2.common.grpc.Direction as ProtoDirection -fun GroupBatch.toByteArray() = Unpooled.buffer().run { +fun GroupBatch.toByteArray(): ByteArray = Unpooled.buffer().run { GroupBatchCodec.encode(this@toByteArray, this@run) ByteArray(readableBytes()).apply(::readBytes) } +fun ByteBuf.toByteArray(): ByteArray = ByteArray(readableBytes()).apply(::readBytes) + val Direction.proto: ProtoDirection get() = when (this) { Direction.INCOMING -> ProtoDirection.FIRST @@ -53,6 +59,29 @@ fun Collection.filter(batch: GroupBatch): GroupBatch? { return null } +fun EventId.toProto(): EventID = EventID.newBuilder().also { + it.id = id + it.bookName = book + it.scope = scope + it.startTimestamp = timestamp.toTimestamp() +}.build() + +fun MessageId.toProto(book: String, sessionGroup: String): MessageID = MessageID.newBuilder().also { + it.bookName = book + it.direction = if (direction == Direction.INCOMING) com.exactpro.th2.common.grpc.Direction.FIRST else com.exactpro.th2.common.grpc.Direction.SECOND + it.sequence = sequence + it.timestamp = timestamp.toTimestamp() + + it.addAllSubsequence(subsequence) + + it.connectionIdBuilder.also { connectionId -> + connectionId.sessionGroup = sessionGroup.ifBlank { sessionAlias } + connectionId.sessionAlias = sessionAlias + } +}.build() + +fun MessageId.toProto(groupBatch: GroupBatch): MessageID = toProto(groupBatch.book, groupBatch.sessionGroup) + private fun Collection?.verify(value: String): Boolean { if (isNullOrEmpty()) { return true } return all { it.checkFieldValue(value) } From 9449978252df2578ced1a62f5b431e5de597ee14 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 26 Apr 2023 13:30:03 +0400 Subject: [PATCH 121/154] Reset RawMessage.body after byte array transformation --- .../impl/rabbitmq/transport/TransportUtils.kt | 57 ++++++++++++++----- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt index cbf59c5f2..206b5e922 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt @@ -16,13 +16,21 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport +import com.exactpro.th2.common.grpc.Direction.FIRST +import com.exactpro.th2.common.grpc.Direction.SECOND import com.exactpro.th2.common.grpc.EventID import com.exactpro.th2.common.grpc.MessageID import com.exactpro.th2.common.message.toTimestamp -import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.* +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.BOOK_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.DIRECTION_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.checkFieldValue import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.Direction.INCOMING +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.Direction.OUTGOING import io.netty.buffer.ByteBuf import io.netty.buffer.Unpooled import com.exactpro.th2.common.grpc.Direction as ProtoDirection @@ -32,12 +40,13 @@ fun GroupBatch.toByteArray(): ByteArray = Unpooled.buffer().run { ByteArray(readableBytes()).apply(::readBytes) } -fun ByteBuf.toByteArray(): ByteArray = ByteArray(readableBytes()).apply(::readBytes) +fun ByteBuf.toByteArray(): ByteArray = ByteArray(readableBytes()) + .apply(::readBytes).also { resetReaderIndex() } val Direction.proto: ProtoDirection get() = when (this) { - Direction.INCOMING -> ProtoDirection.FIRST - Direction.OUTGOING -> ProtoDirection.SECOND + INCOMING -> FIRST + OUTGOING -> SECOND } fun Collection.filter(batch: GroupBatch): GroupBatch? { @@ -46,12 +55,22 @@ fun Collection.filter(batch: GroupBatch): GroupBatch? { } forEach { filterSet -> - if (!filterSet.metadata[BOOK_KEY].verify(batch.book)) { return@forEach } - if (!filterSet.metadata[SESSION_GROUP_KEY].verify(batch.sessionGroup)) { return@forEach } - - if (!filterSet.metadata[SESSION_ALIAS_KEY].verify(batch.groups) { id.sessionAlias }) { return@forEach } - if (!filterSet.metadata[MESSAGE_TYPE_KEY].verify(batch.groups) { if (this is ParsedMessage) type else "" }) { return@forEach } - if (!filterSet.metadata[DIRECTION_KEY].verify(batch.groups) { id.direction.proto.name }) { return@forEach } + if (!filterSet.metadata[BOOK_KEY].verify(batch.book)) { + return@forEach + } + if (!filterSet.metadata[SESSION_GROUP_KEY].verify(batch.sessionGroup)) { + return@forEach + } + + if (!filterSet.metadata[SESSION_ALIAS_KEY].verify(batch.groups) { id.sessionAlias }) { + return@forEach + } + if (!filterSet.metadata[MESSAGE_TYPE_KEY].verify(batch.groups) { if (this is ParsedMessage) type else "" }) { + return@forEach + } + if (!filterSet.metadata[DIRECTION_KEY].verify(batch.groups) { id.direction.proto.name }) { + return@forEach + } return batch } @@ -68,7 +87,7 @@ fun EventId.toProto(): EventID = EventID.newBuilder().also { fun MessageId.toProto(book: String, sessionGroup: String): MessageID = MessageID.newBuilder().also { it.bookName = book - it.direction = if (direction == Direction.INCOMING) com.exactpro.th2.common.grpc.Direction.FIRST else com.exactpro.th2.common.grpc.Direction.SECOND + it.direction = if (direction == INCOMING) FIRST else SECOND it.sequence = sequence it.timestamp = timestamp.toTimestamp() @@ -83,7 +102,9 @@ fun MessageId.toProto(book: String, sessionGroup: String): MessageID = MessageID fun MessageId.toProto(groupBatch: GroupBatch): MessageID = toProto(groupBatch.book, groupBatch.sessionGroup) private fun Collection?.verify(value: String): Boolean { - if (isNullOrEmpty()) { return true } + if (isNullOrEmpty()) { + return true + } return all { it.checkFieldValue(value) } } @@ -91,12 +112,18 @@ private inline fun Collection?.verify( messageGroups: Collection, value: Message<*>.() -> String ): Boolean { - if (isNullOrEmpty()) { return true } + if (isNullOrEmpty()) { + return true + } // Illegal cases when groups or messages are empty - if (messageGroups.isEmpty()) { return false } + if (messageGroups.isEmpty()) { + return false + } val firstGroup = messageGroups.first() - if (firstGroup.messages.isEmpty()) { return false } + if (firstGroup.messages.isEmpty()) { + return false + } return all { filter -> filter.checkFieldValue(firstGroup.messages.first().value()) } } \ No newline at end of file From 53ce3b5b1fb44e7f6904de7b250c35add36931ce Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Wed, 7 Jun 2023 17:16:03 +0400 Subject: [PATCH 122/154] Update cradle version (#268) * Update the version of the cradle The previous dev version has DataStax classes in API. The current one does not. It breaks compatibility when we use common with new Cradle versions. * Build dev release on tag push --- .github/workflows/dev-release-java-publish-sonatype.yml | 7 ++----- README.md | 6 ++++++ build.gradle | 5 ++++- gradle.properties | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev-release-java-publish-sonatype.yml b/.github/workflows/dev-release-java-publish-sonatype.yml index 5d2433d96..0c23d75c8 100644 --- a/.github/workflows/dev-release-java-publish-sonatype.yml +++ b/.github/workflows/dev-release-java-publish-sonatype.yml @@ -1,12 +1,9 @@ name: Build and publish dev-release Java distributions to sonatype. on: - workflow_dispatch: push: - branches: - - dev-version-* - paths: - - gradle.properties + tags: + - \d+.\d+.\d+-dev jobs: build: diff --git a/README.md b/README.md index 59c1c936a..eb9f7f38b 100644 --- a/README.md +++ b/README.md @@ -361,6 +361,12 @@ dependencies { ## Release notes +### 5.2.1 + +#### Changed: + ++ The Cradle version is update to 5.0.2-dev-*. + ### 5.2.0 + Merged with 3.44.1 diff --git a/build.gradle b/build.gradle index e47d9d00c..388930bb4 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.0.0-dev-version-5-2567311063-SNAPSHOT' + cradleVersion = '5.0.2-dev' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } @@ -204,6 +204,9 @@ dependencies { implementation "org.apache.commons:commons-lang3" implementation "org.apache.commons:commons-collections4" implementation "org.apache.commons:commons-text" + implementation("commons-io:commons-io") { + because('we need FilenameUtil to use wildcard matching') + } implementation "commons-cli:commons-cli" implementation "com.fasterxml.jackson.core:jackson-core" diff --git a/gradle.properties b/gradle.properties index 7d9fed0c3..b8fc06ddf 100644 --- a/gradle.properties +++ b/gradle.properties @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.2.0 +release_version=5.2.1 description = 'th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j From bddfde94c90066fa870095a3fa399f02117db9e0 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Thu, 15 Jun 2023 12:41:44 +0400 Subject: [PATCH 123/154] Json raw body (#266) Builders were added to make the transport objects immutable (in terms of making all fields final) * Make body deserialization lazy * Make parsed message fully immutable * Remove soft clean api. Migrate to JSON as parsed body * [TH2-4907] Added utils to convert proto to transport * Add support for nullable values in parsed body * [TS-1450] Added transport to proto and back methods * Extended CollectionBuilder, Message.Builder, MessageUtils, ValueUtils * addMetadataProperty and addField are extracted to extension methods * Moved protbuf message toTreeTable to common-utils * Moved some methods from TransportUtils to common-utils --------- Co-authored-by: nikita.smirnov Co-authored-by: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Co-authored-by: Oleg Smelov Co-authored-by: dilyaver.kaibulaev --- README.md | 337 ++++++++++++------ build.gradle | 122 ++++--- gradle.properties | 6 +- .../builders/GenericCollectionBuilder.java | 45 +++ .../th2/common/message/MessageFilterUtils.kt | 74 ++-- .../th2/common/message/MessageUtils.kt | 136 ++++--- .../impl/rabbitmq/transport/Cleanable.kt | 6 - .../message/impl/rabbitmq/transport/Codecs.kt | 137 ++++--- .../impl/rabbitmq/transport/EventId.kt | 33 +- .../impl/rabbitmq/transport/GroupBatch.kt | 32 +- .../impl/rabbitmq/transport/Message.kt | 30 +- .../impl/rabbitmq/transport/MessageGroup.kt | 44 ++- .../impl/rabbitmq/transport/MessageId.kt | 53 +-- .../impl/rabbitmq/transport/ParsedMessage.kt | 308 +++++++++++++--- .../impl/rabbitmq/transport/RawMessage.kt | 55 +-- .../impl/rabbitmq/transport/TransportUtils.kt | 63 ++-- .../transport/builders/CollectionBuilder.kt | 42 +++ .../rabbitmq/transport/builders/MapBuilder.kt | 31 ++ .../exactpro/th2/common/value/ValueUtils.kt | 117 ++++-- .../impl/rabbitmq/transport/CleanableTest.kt | 169 --------- .../impl/rabbitmq/transport/CodecsTest.kt | 40 ++- ...ransportGroupBatchRouterIntegrationTest.kt | 11 +- .../TransportGroupBatchRouterTest.kt | 15 +- .../rabbitmq/transport/TransportUtilsTest.kt | 184 +++++++--- 24 files changed, 1326 insertions(+), 764 deletions(-) create mode 100644 src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/GenericCollectionBuilder.java create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/CollectionBuilder.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/MapBuilder.kt delete mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt diff --git a/README.md b/README.md index 59c1c936a..b65c72e32 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,15 @@ -# th2 common library (Java) (5.2.0) +# th2 common library (Java) (5.3.0) ## Usage Firstly, you must import CommonFactory class: + ``` import com.exactpro.th2.common.schema.factory.CommonFactory ``` + Then you will create an instance of imported class, by choosing one of the following options: + 1. Create factory with configs from the default path (`/var/th2/config/*`): ``` var factory = CommonFactory(); @@ -19,10 +22,10 @@ Then you will create an instance of imported class, by choosing one of the follo ``` var factory = CommonFactory.createFromArguments(args); ``` - You can use one of the following groups of arguments. Arguments from different - groups cannot be used together. - - The first group: + You can use one of the following groups of arguments. Arguments from different + groups cannot be used together. + + The first group: * --rabbitConfiguration - path to json file with RabbitMQ configuration * --messageRouterConfiguration - path to json file with configuration for MessageRouter * --grpcRouterConfiguration - path to json file with configuration for GrpcRouter @@ -32,53 +35,69 @@ Then you will create an instance of imported class, by choosing one of the follo * --prometheusConfiguration - path to json file with configuration for prometheus metrics server * --boxConfiguration - path to json file with boxes configuration and information * -c/--configs - folder with json files for schemas configurations with special names: + 1. rabbitMq.json - configuration for RabbitMQ 2. mq.json - configuration for MessageRouter 3. grpc.json - configuration for GrpcRouter 4. cradle.json - configuration for cradle 5. custom.json - custom configuration - - The second group: + + The second group: * --namespace - the namespace in Kubernetes to search config maps * --boxName - the name of the target th2 box placed in the specified Kubernetes namespace * --contextName - the context name to search connect parameters in Kube config - * --dictionaries - the mapping between a dictionary in infra schema and a dictionary type in the format: - `--dictionaries =[ =]`. - It can be useful when you required dictionaries to start a specific box. - - Their usage is disclosed further. - + * --dictionaries - the mapping between a dictionary in infra schema and a dictionary type in the format: + `--dictionaries =[ =]`. + It can be useful when you required dictionaries to start a specific box. + + Their usage is disclosed further. + 1. Create factory with a namespace in Kubernetes and the name of the target th2 box from Kubernetes: ``` var factory = CommonFactory.createFromKubernetes(namespace, boxName); ``` - It also can be called by using `createFromArguments(args)` with arguments `--namespace` and `--boxName`. -1. Create factory with a namespace in Kubernetes, the name of the target th2 box from Kubernetes and the name of context to choose from Kube config: + It also can be called by using `createFromArguments(args)` with arguments `--namespace` and `--boxName`. +1. Create factory with a namespace in Kubernetes, the name of the target th2 box from Kubernetes and the name of context + to choose from Kube config: ``` var factory = CommonFactory.createFromKubernetes(namespace, boxName, contextName); ``` - It also can be called by using `createFromArguments(args)` with arguments `--namespace`, `--boxName` and `--contextName`. - ContextName parameter is `@Nullable`; if it is set to null, the current context will not be changed. + It also can be called by using `createFromArguments(args)` with arguments `--namespace`, `--boxName` + and `--contextName`. + ContextName parameter is `@Nullable`; if it is set to null, the current context will not be changed. ### Configuration formats The `CommonFactory` reads a RabbitMQ configuration from the rabbitMQ.json file. + * host - the required setting defines the RabbitMQ host. -* vHost - the required setting defines the virtual host that will be used for connecting to RabbitMQ. +* vHost - the required setting defines the virtual host that will be used for connecting to RabbitMQ. Please see more details about the virtual host in RabbitMQ via [link](https://www.rabbitmq.com/vhosts.html) * port - the required setting defines the RabbitMQ port. -* username - the required setting defines the RabbitMQ username. +* username - the required setting defines the RabbitMQ username. The user must have permission to publish messages via routing keys and subscribe to message queues. * password - the required setting defines the password that will be used for connecting to RabbitMQ. -* exchangeName - the required setting defines the exchange that will be used for sending/subscribing operation in MQ routers. - Please see more details about the exchanges in RabbitMQ via [link](https://www.rabbitmq.com/tutorials/amqp-concepts.html#exchanges) -* connectionTimeout - the connection TCP establishment timeout in milliseconds with its default value set to 60000. Use zero for infinite waiting. -* connectionCloseTimeout - the timeout in milliseconds for completing all the close-related operations, use -1 for infinity, the default value is set to 10000. -* maxRecoveryAttempts - this option defines the number of reconnection attempts to RabbitMQ, with its default value set to 5. - The `th2_readiness` probe is set to false and publishers are blocked after a lost connection to RabbitMQ. The `th2_readiness` probe is reverted to true if the connection will be recovered during specified attempts otherwise the `th2_liveness` probe will be set to false. -* minConnectionRecoveryTimeout - this option defines a minimal interval in milliseconds between reconnect attempts, with its default value set to 10000. Common factory increases the reconnect interval values from minConnectionRecoveryTimeout to maxConnectionRecoveryTimeout. -* maxConnectionRecoveryTimeout - this option defines a maximum interval in milliseconds between reconnect attempts, with its default value set to 60000. Common factory increases the reconnect interval values from minConnectionRecoveryTimeout to maxConnectionRecoveryTimeout. -* prefetchCount - this option is the maximum number of messages that the server will deliver, with its value set to 0 if unlimited, the default value is set to 10. +* exchangeName - the required setting defines the exchange that will be used for sending/subscribing operation in MQ + routers. + Please see more details about the exchanges in RabbitMQ + via [link](https://www.rabbitmq.com/tutorials/amqp-concepts.html#exchanges) +* connectionTimeout - the connection TCP establishment timeout in milliseconds with its default value set to 60000. Use + zero for infinite waiting. +* connectionCloseTimeout - the timeout in milliseconds for completing all the close-related operations, use -1 for + infinity, the default value is set to 10000. +* maxRecoveryAttempts - this option defines the number of reconnection attempts to RabbitMQ, with its default value set + to 5. + The `th2_readiness` probe is set to false and publishers are blocked after a lost connection to RabbitMQ. + The `th2_readiness` probe is reverted to true if the connection will be recovered during specified attempts otherwise + the `th2_liveness` probe will be set to false. +* minConnectionRecoveryTimeout - this option defines a minimal interval in milliseconds between reconnect attempts, with + its default value set to 10000. Common factory increases the reconnect interval values from + minConnectionRecoveryTimeout to maxConnectionRecoveryTimeout. +* maxConnectionRecoveryTimeout - this option defines a maximum interval in milliseconds between reconnect attempts, with + its default value set to 60000. Common factory increases the reconnect interval values from + minConnectionRecoveryTimeout to maxConnectionRecoveryTimeout. +* prefetchCount - this option is the maximum number of messages that the server will deliver, with its value set to 0 if + unlimited, the default value is set to 10. * messageRecursionLimit - an integer number denotes how deep the nested protobuf message might be, set by default 100 ```json @@ -100,6 +119,7 @@ The `CommonFactory` reads a RabbitMQ configuration from the rabbitMQ.json file. ``` The `CommonFactory` reads a message's router configuration from the `mq.json` file. + * queues - the required settings defines all pins for an application * name - routing key in RabbitMQ for sending * queue - queue's name in RabbitMQ for subscribe @@ -108,12 +128,12 @@ The `CommonFactory` reads a message's router configuration from the `mq.json` fi * first * second * subscribe - * publish + * publish * parsed * raw * store * event - + * filters - pin's message's filters * metadata - a metadata filters * message - a message's fields filters @@ -121,7 +141,8 @@ The `CommonFactory` reads a message's router configuration from the `mq.json` fi * globalNotification - notification exchange in RabbitMQ * exchange - `global-notification` by default -Filters format: +Filters format: + * fieldName - a field's name * expectedValue - expected field's value (not used for all operations) * operation - operation's type @@ -167,9 +188,13 @@ Filters format: ``` The `CommonFactory` reads a gRPC router configuration from the `grpc_router.json` file. -* enableSizeMeasuring - this option enables the gRPC message size measuring. Please note the feature decreases gRPC throughput. Default value is false. + +* enableSizeMeasuring - this option enables the gRPC message size measuring. Please note the feature decreases gRPC + throughput. Default value is false. * keepAliveInterval - number of seconds between keep alive messages. Default value is 60 -* maxMessageSize - this option enables endpoint message filtering based on message size (message with size larger than option value will be skipped). By default, it has a value of `4 MB`. The unit of measurement of the value is number of bytes. +* maxMessageSize - this option enables endpoint message filtering based on message size (message with size larger than + option value will be skipped). By default, it has a value of `4 MB`. The unit of measurement of the value is number of + bytes. ```json { @@ -180,6 +205,7 @@ The `CommonFactory` reads a gRPC router configuration from the `grpc_router.json ``` The `CommonFactory` reads a gRPC configuration from the `grpc.json` file. + * services - grpc services configurations * server - grpc server configuration * endpoint - grpc endpoint configuration @@ -215,16 +241,22 @@ The `CommonFactory` reads a gRPC configuration from the `grpc.json` file. ``` The `CommonFactory` reads a Cradle configuration from the cradle.json file. + * dataCenter - the required setting defines the data center in the Cassandra cluster. * host - the required setting defines the Cassandra host. * port - the required setting defines the Cassandra port. * keyspace - the required setting defines the keyspace (top-level database object) in the Cassandra data center. -* username - the required setting defines the Cassandra username. The user must have permission to write data using a specified keyspace. +* username - the required setting defines the Cassandra username. The user must have permission to write data using a + specified keyspace. * password - the required setting defines the password that will be used for connecting to Cassandra. -* cradleMaxEventBatchSize - this option defines the maximum event batch size in bytes with its default value set to 1048576. -* cradleMaxMessageBatchSize - this option defines the maximum message batch size in bytes with its default value set to 1048576. -* timeout - this option defines connection timeout in milliseconds. If set to 0 or omitted, the default value of 5000 is used. -* pageSize - this option defines the size of the result set to fetch at a time. If set to 0 or omitted, the default value of 5000 is used. +* cradleMaxEventBatchSize - this option defines the maximum event batch size in bytes with its default value set to + 1048576. +* cradleMaxMessageBatchSize - this option defines the maximum message batch size in bytes with its default value set to + 1048576. +* timeout - this option defines connection timeout in milliseconds. If set to 0 or omitted, the default value of 5000 is + used. +* pageSize - this option defines the size of the result set to fetch at a time. If set to 0 or omitted, the default + value of 5000 is used. * prepareStorage - enables database schema initialization if Cradle is used. By default, it has a value of `false` ```json @@ -245,49 +277,78 @@ The `CommonFactory` reads a Cradle configuration from the cradle.json file. ### Requirements for creating factory with Kubernetes -1. It is necessary to have Kubernetes configuration written in ~/.kube/config. See more on kubectl configuration [here](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/). +1. It is necessary to have Kubernetes configuration written in ~/.kube/config. See more on kubectl + configuration [here](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/). -1. Also note that `generated_configs` directory will be created to store `.json` files with configs from Kubernetes. Those files are overridden when `CommonFactory.createFromKubernetes(namespace, boxName)` and `CommonFactory.createFromKubernetes(namespace, boxName, contextName)` are invoked again. +1. Also note that `generated_configs` directory will be created to store `.json` files with configs from Kubernetes. + Those files are overridden when `CommonFactory.createFromKubernetes(namespace, boxName)` + and `CommonFactory.createFromKubernetes(namespace, boxName, contextName)` are invoked again. -1. Users need to have authentication with the service account token that has the necessary access to read CRs and secrets from the specified namespace. +1. Users need to have authentication with the service account token that has the necessary access to read CRs and + secrets from the specified namespace. After that you can receive various Routers through factory properties: + ``` -var messageRouter = factory.getMessageRouterParsedBatch(); -var rawRouter = factory.getMessageRouterRawBatch(); -var eventRouter = factory.getEventBatchRouter(); + +var protoMessageGroupRouter = factory.getMessageRouterMessageGroupBatch(); +var protoMessageRouter = factory.getMessageRouterParsedBatch(); +var protoRawRouter = factory.getMessageRouterRawBatch(); +var protoEventRouter = factory.getEventBatchRouter(); + +var transportGroupRouter = factory.getTransportGroupBatchRouter(); ``` -`messageRouter` is working with `MessageBatch`
-`rawRouter` is working with `RawMessageBatch`
-`eventRouter` is working with `EventBatch` +`protoMessageGroupRouter` is working with `MessageGroupBatch`
+`protoMessageRouter` is working with `MessageBatch`
+`protoRawRouter` is working with `RawMessageBatch`
+`protoEventRouter` is working with `EventBatch`
+ +`transportGroupRouter` is working with `GroupBatch`
+ +Note: MessageRouterParsedBatch and MessageRouterRawBatch are not recommended to use because they are adapters for +MessageRouterMessageGroupBatch and execute additional repacking + +Please refer +to [th2-grpc-common](https://github.com/th2-net/th2-grpc-common/blob/master/src/main/proto/th2_grpc_common/common.proto "common.proto") +for further details. -Please refer to [th2-grpc-common](https://github.com/th2-net/th2-grpc-common/blob/master/src/main/proto/th2_grpc_common/common.proto "common.proto") for further details. +With the router created, you can subscribe to pins (by specifying the callback function) or to send data that the router +works with: -With the router created, you can subscribe to pins (by specifying the callback function) or to send data that the router works with: ``` router.subscribe(callback) # subscribe to only one pin router.subscribeAll(callback) # subscribe to one or several pins router.send(message) # send to only one pin router.sendAll(message) # send to one or several pins ``` + You can perform these actions by providing pin attributes in addition to the default ones. + ``` router.subscribe(callback, attrs...) # subscribe to only one pin router.subscribeAll(callback, attrs...) # subscribe to one or several pins router.send(message, attrs...) # send to only one pin router.sendAll(message, attrs...) # send to one or several pins ``` + The default attributes are: -- `message_parsed_batch_router` + +- `proto_message_group_batch_router` + - Subscribe: `subscribe` + - Send: `publish` +- `proto_message_parsed_batch_router` - Subscribe: `subscribe`, `parsed` - Send: `publish`, `parsed` -- `message_raw_batch_router` +- `proto_message_raw_batch_router` - Subscribe: `subscribe`, `raw` - Send: `publish`, `raw` -- `event_batch_router` +- `proto_event_batch_router` - Subscribe: `subscribe`, `event` - Send: `publish`, `event` +- `transport_group_message_batch_router` + - Subscribe: `subscribe`, `transport-group` + - Send: `publish`, `transport-group` This library allows you to: @@ -299,11 +360,15 @@ This kind of router provides the ability for boxes to interact between each othe #### Server -gRPC router rises a gRPC server with enabled [grpc-service-reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md#grpc-server-reflection-tutorial) since the 3.38.0 version. -It means that the users can execute calls from the console or through scripts via [grpcurl](https://github.com/fullstorydev/grpcurl#grpcurl) without gRPC schema (files with proto extensions describes gRPC service structure) +gRPC router rises a gRPC server with +enabled [grpc-service-reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md#grpc-server-reflection-tutorial) +since the 3.38.0 version. +It means that the users can execute calls from the console or through scripts +via [grpcurl](https://github.com/fullstorydev/grpcurl#grpcurl) without gRPC schema (files with proto extensions +describes gRPC service structure) ## Export common metrics to Prometheus - + It can be performed by the following utility methods in CommonMetrics class * `setLiveness` - sets "liveness" metric of a service (exported as `th2_liveness` gauge) @@ -313,31 +378,59 @@ NOTES: * in order for the metrics to be exported, you will also need to create an instance of CommonFactory * common JVM metrics will also be exported alongside common service metrics -* some metric labels are enumerations (`th2_type`: `MESSAGE_GROUP`, `EVENT`, ``;`message_type`: `RAW_MESSAGE`, `MESSAGE`) +* some metric labels are + enumerations (`th2_type`: `MESSAGE_GROUP`, `EVENT`, ``;`message_type`: `RAW_MESSAGE`, `MESSAGE`) RABBITMQ METRICS: -* th2_rabbitmq_message_size_publish_bytes (`th2_pin`, `th2_type`, `exchange`, `routing_key`): number of published message bytes to RabbitMQ. The intended is intended for any data transferred via RabbitMQ, for example, th2 batch message or event or custom content -* th2_rabbitmq_message_publish_total (`th2_pin`, `th2_type`, `exchange`, `routing_key`): quantity of published messages to RabbitMQ. The intended is intended for any data transferred via RabbitMQ, for example, th2 batch message or event or custom content -* th2_rabbitmq_message_size_subscribe_bytes (`th2_pin`, `th2_type`, `queue`): number of bytes received from RabbitMQ, it includes bytes of messages dropped after filters. For information about the number of dropped messages, please refer to 'th2_message_dropped_subscribe_total' and 'th2_message_group_dropped_subscribe_total'. The message is intended for any data transferred via RabbitMQ, for example, th2 batch message or event or custom content -* th2_rabbitmq_message_process_duration_seconds (`th2_pin`, `th2_type`, `queue`): time of message processing during subscription from RabbitMQ in seconds. The message is intended for any data transferred via RabbitMQ, for example, th2 batch message or event or custom content + +* th2_rabbitmq_message_size_publish_bytes (`th2_pin`, `th2_type`, `exchange`, `routing_key`): number of published + message bytes to RabbitMQ. The intended is intended for any data transferred via RabbitMQ, for example, th2 batch + message or event or custom content +* th2_rabbitmq_message_publish_total (`th2_pin`, `th2_type`, `exchange`, `routing_key`): quantity of published messages + to RabbitMQ. The intended is intended for any data transferred via RabbitMQ, for example, th2 batch message or event + or custom content +* th2_rabbitmq_message_size_subscribe_bytes (`th2_pin`, `th2_type`, `queue`): number of bytes received from RabbitMQ, it + includes bytes of messages dropped after filters. For information about the number of dropped messages, please refer + to 'th2_message_dropped_subscribe_total' and 'th2_message_group_dropped_subscribe_total'. The message is intended for + any data transferred via RabbitMQ, for example, th2 batch message or event or custom content +* th2_rabbitmq_message_process_duration_seconds (`th2_pin`, `th2_type`, `queue`): time of message processing during + subscription from RabbitMQ in seconds. The message is intended for any data transferred via RabbitMQ, for example, th2 + batch message or event or custom content GRPC METRICS: -* th2_grpc_invoke_call_total (`th2_pin`, `service_name`, `service_method`): total number of calling particular gRPC method -* th2_grpc_invoke_call_request_bytes (`th2_pin`, `service_name`, `service_method`): number of bytes sent to particular gRPC call -* th2_grpc_invoke_call_response_bytes (`th2_pin`, `service_name`, `service_method`): number of bytes sent to particular gRPC call -* th2_grpc_receive_call_total (`th2_pin`, `service_name`, `service_method`): total number of consuming particular gRPC method -* th2_grpc_receive_call_request_bytes (`th2_pin`, `service_name`, `service_method`): number of bytes received from particular gRPC call -* th2_grpc_receive_call_response_bytes (`th2_pin`, `service_name`, `service_method`): number of bytes sent to particular gRPC call + +* th2_grpc_invoke_call_total (`th2_pin`, `service_name`, `service_method`): total number of calling particular gRPC + method +* th2_grpc_invoke_call_request_bytes (`th2_pin`, `service_name`, `service_method`): number of bytes sent to particular + gRPC call +* th2_grpc_invoke_call_response_bytes (`th2_pin`, `service_name`, `service_method`): number of bytes sent to particular + gRPC call +* th2_grpc_receive_call_total (`th2_pin`, `service_name`, `service_method`): total number of consuming particular gRPC + method +* th2_grpc_receive_call_request_bytes (`th2_pin`, `service_name`, `service_method`): number of bytes received from + particular gRPC call +* th2_grpc_receive_call_response_bytes (`th2_pin`, `service_name`, `service_method`): number of bytes sent to particular + gRPC call MESSAGES METRICS: -* th2_message_publish_total (`th2_pin`, `session_alias`, `direction`, `message_type`): quantity of published raw or parsed messages -* th2_message_subscribe_total (`th2_pin`, `session_alias`, `direction`, `message_type`): quantity of received raw or parsed messages, includes dropped after filters. For information about the number of dropped messages, please refer to 'th2_message_dropped_subscribe_total' -* th2_message_dropped_publish_total (`th2_pin`, `session_alias`, `direction`, `message_type`): quantity of published raw or parsed messages dropped after filters -* th2_message_dropped_subscribe_total (`th2_pin`, `session_alias`, `direction`, `message_type`): quantity of received raw or parsed messages dropped after filters -* th2_message_group_publish_total (`th2_pin`, `session_alias`, `direction`): quantity of published message groups -* th2_message_group_subscribe_total (`th2_pin`, `session_alias`, `direction`): quantity of received message groups, includes dropped after filters. For information about the number of dropped messages, please refer to 'th2_message_group_dropped_subscribe_total' -* th2_message_group_dropped_publish_total (`th2_pin`, `session_alias`, `direction`): quantity of published message groups dropped after filters -* th2_message_group_dropped_subscribe_total (`th2_pin`, `session_alias`, `direction`): quantity of received message groups dropped after filters + +* th2_message_publish_total (`th2_pin`, `session_alias`, `direction`, `message_type`): quantity of published raw or + parsed messages +* th2_message_subscribe_total (`th2_pin`, `session_alias`, `direction`, `message_type`): quantity of received raw or + parsed messages, includes dropped after filters. For information about the number of dropped messages, please refer + to 'th2_message_dropped_subscribe_total' +* th2_message_dropped_publish_total (`th2_pin`, `session_alias`, `direction`, `message_type`): quantity of published raw + or parsed messages dropped after filters +* th2_message_dropped_subscribe_total (`th2_pin`, `session_alias`, `direction`, `message_type`): quantity of received + raw or parsed messages dropped after filters +* th2_message_group_publish_total (`th2_pin`, `session_alias`, `direction`): quantity of published message groups +* th2_message_group_subscribe_total (`th2_pin`, `session_alias`, `direction`): quantity of received message groups, + includes dropped after filters. For information about the number of dropped messages, please refer to ' + th2_message_group_dropped_subscribe_total' +* th2_message_group_dropped_publish_total (`th2_pin`, `session_alias`, `direction`): quantity of published message + groups dropped after filters +* th2_message_group_dropped_subscribe_total (`th2_pin`, `session_alias`, `direction`): quantity of received message + groups dropped after filters * th2_message_group_sequence_publish (`th2_pin`, `session_alias`, `direction`): last published sequence * th2_message_group_sequence_subscribe (`th2_pin`, `session_alias`, `direction`): last received sequence @@ -348,7 +441,8 @@ EVENTS METRICS: ### Test extensions: -To be able to use test extensions please fill build.gradle as in the example below: +To be able to use test extensions please fill build.gradle as in the example below: + ```groovy plugins { id 'java-test-fixtures' @@ -361,6 +455,10 @@ dependencies { ## Release notes +### 5.3.0 + ++ Implemented message routers used th2 transport protocol for interaction + ### 5.2.0 + Merged with 3.44.1 @@ -388,17 +486,20 @@ dependencies { --- ### 3.44.1 -+ Remove unused dependency + ++ Remove unused dependency + Updated bom:4.2.0 ### 3.43.0 + + There is no support for log4j version 1. + Work was done to eliminate vulnerabilities in _common_ and _bom_ dependencies. + **th2-bom must be updated to _4.0.3_ or higher.** ### 3.42.0 -+ Added the `enableSizeMeasuring`, `maxMessageSize`, `keepAliveInterval` options into gRPC router configuration. - Default values are false, 4194304, 60 + ++ Added the `enableSizeMeasuring`, `maxMessageSize`, `keepAliveInterval` options into gRPC router configuration. + Default values are false, 4194304, 60 ### 3.41.1 @@ -407,12 +508,13 @@ dependencies { ### 3.41.0 -+ Work was done to eliminate vulnerabilities in _common_ and _bom_ dependencies. - + **th2-bom must be updated to _4.0.1_ or higher.** ++ Work was done to eliminate vulnerabilities in _common_ and _bom_ dependencies. + + **th2-bom must be updated to _4.0.1_ or higher.** ### 3.40.0 -+ gRPC router creates server support [grpc-service-reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md#grpc-server-reflection-tutorial) ++ gRPC router creates server + support [grpc-service-reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md#grpc-server-reflection-tutorial) ### 3.39.3 @@ -422,12 +524,13 @@ dependencies { ### 3.39.2 + Fixed: - + The message is not confirmed when it was filtered + + The message is not confirmed when it was filtered ### 3.39.1 + Fixed: - + cradle version updated to 3.1.2. Unicode character sizes are calculated properly during serialization and tests are added for it also. + + cradle version updated to 3.1.2. Unicode character sizes are calculated properly during serialization and tests + are added for it also. ### 3.39.0 @@ -436,8 +539,8 @@ dependencies { ### 3.38.0 + Migration to log4j2. **th2-bom must be updated to _3.2.0_ or higher** - + There is backward compatibility with the log4j format. - + If both configurations are available, log4j2 is preferable. + + There is backward compatibility with the log4j format. + + If both configurations are available, log4j2 is preferable. ### 3.37.2 @@ -446,7 +549,7 @@ dependencies { ### 3.37.1 + Fixed: - + When creating the `CommonFactory` from k8s the logging configuration wouldn't be downloaded + + When creating the `CommonFactory` from k8s the logging configuration wouldn't be downloaded ### 3.37.0 @@ -473,12 +576,13 @@ dependencies { #### Added: + Methods for subscription with manual acknowledgement - (if the **prefetch count** is requested and no messages are acknowledged the reading from the queue will be suspended). + (if the **prefetch count** is requested and no messages are acknowledged the reading from the queue will be + suspended). Please, note that only one subscriber with manual acknowledgement can be subscribed to a queue ### 3.32.1 -+ Fixed: gRPC router didn't shut down underlying Netty's EventLoopGroup and ExecutorService ++ Fixed: gRPC router didn't shut down underlying Netty's EventLoopGroup and ExecutorService ### 3.32.0 @@ -501,6 +605,7 @@ dependencies { + Update grpc-common from 3.8.0 to 3.9.0 ### 3.31.1 + + Feature as test assertion methods for messages from fixtures ### 3.31.0 @@ -535,24 +640,30 @@ dependencies { + Added new metrics and removed old ### 3.26.5 + + Migrated `grpc-common` version from `3.7.0` to `3.8.0` + Added `time_precision` and `decimal_precision` parameters to `RootComparisonSettings` ### 3.26.4 + + Migrated `grpc-common` version from `3.6.0` to `3.7.0` - + Added `check_repeating_group_order` parameter to `RootComparisonSettings` message + + Added `check_repeating_group_order` parameter to `RootComparisonSettings` message ### 3.26.3 + + Migrated `grpc-common` version from `3.5.0` to `3.6.0` - + Added `description` parameter to `RootMessageFilter` message + + Added `description` parameter to `RootMessageFilter` message ### 3.26.2 + + Fix `SimpleFilter` and `ValueFilter` treeTable convertation ### 3.26.1 + + Add `SimpleList` display to `TreeTableEntry`; ### 3.26.0 + + Update the grpc-common version to 3.5.0 + Added `SimpleList` to `SimpleFilter` @@ -563,21 +674,29 @@ dependencies { ### 3.25.1 #### Changed: -+ Extension method for `MessageRouter` now send the event to all pins that satisfy the requested attributes set + ++ Extension method for `MessageRouter` now send the event to all pins that satisfy the requested attributes + set ### 3.25.0 + + Added util to convert RootMessageFilter into readable collection of payload bodies ### 3.24.2 + + Fixed `messageRecursionLimit` - still was not applied to all kind of RabbitMQ subscribers ### 3.24.1 + + Fixed `messageRecursionLimit` setting for all kind of RabbitMQ subscribers ### 3.24.0 -+ Added setting `messageRecursionLimit`(the default value is set to 100) to RabbitMQ configuration that denotes how deep nested protobuf messages might be. + ++ Added setting `messageRecursionLimit`(the default value is set to 100) to RabbitMQ configuration that denotes how deep + nested protobuf messages might be. ### 3.23.0 + + Update the grpc-common version to 3.4.0 + Added `IN`, `LIKE`, `MORE`, `LESS`, `WILDCARD` FilterOperation and their negative versions @@ -587,13 +706,16 @@ dependencies { + Added `WILDCARD` filter operation, which filter a field by wildcard expression. ### 3.21.2 + + Fixed grpc server start. ### 3.21.1 + + Update the grpc-common version to 3.3.0: - + Added information about message timestamp into the checkpoint + + Added information about message timestamp into the checkpoint ### 3.21.0 + + Added classes for management metrics. + Added the ability for resubscribe on canceled subscriber. @@ -627,10 +749,12 @@ dependencies { + Update Cradle version from `2.9.1` to `2.13.0` ### 3.17.0 + + Extended message utility class - + Added the toRootMessageFilter method to convert a message to root message filter + + Added the toRootMessageFilter method to convert a message to root message filter ### 3.16.5 + + Update `th2-grpc-common` and `th2-grpc-service-generator` versions to `3.2.0` and `3.1.12` respectively ### 3.16.4 @@ -640,7 +764,8 @@ dependencies { ### 3.16.3 + Change the way that channels are stored (they are mapped to the pin instead of to the thread). - It might increase the average number of channels used by the box, but it also limits the max number of channels to the number of pins + It might increase the average number of channels used by the box, but it also limits the max number of channels to the + number of pins ### 3.16.2 @@ -648,13 +773,14 @@ dependencies { **NOTE: one of the methods was not restored and an update to this version might require manual update for your code**. The old methods without `toTraceString` supplier will be removed in the future + Fixed configuration for gRPC server. - + Added the property `workers`, which changes the count of gRPC server's threads - + + Added the property `workers`, which changes the count of gRPC server's threads + ### 3.16.0 + Extended Utility classes - + Added the toTreeTable method to convert message/message filter to event data - + Added the Event.exception method to include an exception and optionally all the causes to the body data as a series of messages + + Added the toTreeTable method to convert message/message filter to event data + + Added the Event.exception method to include an exception and optionally all the causes to the body data as a + series of messages ### 3.15.0 @@ -669,7 +795,7 @@ dependencies { + resets embedded `log4j` configuration before configuring it from a file -### 3.13.5 +### 3.13.5 + fixed a bug with message filtering by `message_type` @@ -687,16 +813,18 @@ dependencies { ### 3.13.1 -+ removed gRPC event loop handling ++ removed gRPC event loop handling + fixed dictionary reading -### 3.13.0 +### 3.13.0 + + reads dictionaries from the /var/th2/config/dictionary folder. + uses mq_router, grpc_router, cradle_manager optional JSON configs from the /var/th2/config folder ### 3.11.0 -+ tries to load log4j.properties files from sources in order: '/var/th2/config', '/home/etc', configured path via cmd, default configuration ++ tries to load log4j.properties files from sources in order: '/var/th2/config', '/home/etc', configured path via cmd, + default configuration ### 3.6.0 @@ -704,4 +832,5 @@ dependencies { ### 3.0.1 -+ metrics related to time measurement of an incoming message handling (Raw / Parsed / Event) migrated to Prometheus [histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) ++ metrics related to time measurement of an incoming message handling (Raw / Parsed / Event) migrated to + Prometheus [histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) diff --git a/build.gradle b/build.gradle index 6e9cdf517..8b5786810 100644 --- a/build.gradle +++ b/build.gradle @@ -21,6 +21,7 @@ plugins { id 'signing' id 'com.google.protobuf' version '0.8.8' apply false id 'org.jetbrains.kotlin.jvm' version "${kotlin_version}" + id 'org.jetbrains.kotlin.kapt' version "${kotlin_version}" id "org.owasp.dependencycheck" version "8.1.1" id "me.champeau.jmh" version "0.6.8" } @@ -29,20 +30,20 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.1.0-separate-executor-4783246997-dda4d35-SNAPSHOT' - junitVersion = '5.8.2' - sharedDir = file("${project.rootDir}/shared") + cradleVersion = '5.1.0-separate-executor-4783246997-dda4d35-SNAPSHOT' + junitVersion = '5.8.2' + sharedDir = file("${project.rootDir}/shared") } repositories { mavenCentral() maven { - name 'Sonatype_snapshots' - url 'https://s01.oss.sonatype.org/content/repositories/snapshots/' + name 'Sonatype_snapshots' + url 'https://s01.oss.sonatype.org/content/repositories/snapshots/' } maven { - name 'Sonatype_releases' - url 'https://s01.oss.sonatype.org/content/repositories/releases/' + name 'Sonatype_releases' + url 'https://s01.oss.sonatype.org/content/repositories/releases/' } mavenLocal() @@ -70,24 +71,25 @@ java { tasks.withType(PublishToMavenRepository) { onlyIf { (repository == publishing.repositories.nexusRepository && - project.hasProperty('nexus_user') && - project.hasProperty('nexus_password') && - project.hasProperty('nexus_url')) || - (repository == publishing.repositories.sonatype && - project.hasProperty('sonatypeUsername') && - project.hasProperty('sonatypePassword')) || - (repository == publishing.repositories.localRepo) + project.hasProperty('nexus_user') && + project.hasProperty('nexus_password') && + project.hasProperty('nexus_url')) || + (repository == publishing.repositories.sonatype && + project.hasProperty('sonatypeUsername') && + project.hasProperty('sonatypePassword')) || + (repository == publishing.repositories.localRepo) } } tasks.withType(Sign) { - onlyIf { project.hasProperty('signingKey') && - project.hasProperty('signingPassword') + onlyIf { + project.hasProperty('signingKey') && + project.hasProperty('signingPassword') } } // disable running task 'initializeSonatypeStagingRepository' on a gitlab -tasks.whenTaskAdded {task -> - if(task.name.equals('initializeSonatypeStagingRepository') && - !(project.hasProperty('sonatypeUsername') && project.hasProperty('sonatypePassword')) +tasks.whenTaskAdded { task -> + if (task.name.equals('initializeSonatypeStagingRepository') && + !(project.hasProperty('sonatypeUsername') && project.hasProperty('sonatypePassword')) ) { task.enabled = false } @@ -98,29 +100,29 @@ publishing { mavenJava(MavenPublication) { from(components.java) pom { - name = rootProject.name - packaging = 'jar' - description = rootProject.description - url = vcs_url - scm { + name = rootProject.name + packaging = 'jar' + description = rootProject.description url = vcs_url - } - licenses { - license { - name = 'The Apache License, Version 2.0' - url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + scm { + url = vcs_url } - } - developers { - developer { - id = 'developer' - name = 'developer' - email = 'developer@exactpro.com' + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + id = 'developer' + name = 'developer' + email = 'developer@exactpro.com' + } + } + scm { + url = vcs_url } - } - scm { - url = vcs_url - } } } } @@ -176,9 +178,15 @@ tasks.register('integrationTest', Test) { dependencies { api platform("com.exactpro.th2:bom:4.2.0") - api 'com.exactpro.th2:grpc-common:4.2.0-dev' - api "com.exactpro.th2:cradle-core:${cradleVersion}" - api 'io.github.microutils:kotlin-logging:2.1.21' + api('com.exactpro.th2:grpc-common:4.2.0-dev') { + because('protobuf transport is main now, this dependnecy should be moved to grpc, mq protobuf modules after splitting') + } + api("com.exactpro.th2:cradle-core:${cradleVersion}") { + because('cradle is included into common library now, this dependnecy should be moved to a cradle module after splitting') + } + api('io.netty:netty-buffer') { + because('th2 transport protocol is included into common library now, this dependnecy should be moved to a th2 transport module after splitting') + } jmh 'org.openjdk.jmh:jmh-core:0.9' jmh 'org.openjdk.jmh:jmh-generator-annprocess:0.9' @@ -187,6 +195,12 @@ dependencies { implementation 'com.exactpro.th2:grpc-service-generator:3.2.2' implementation "com.exactpro.th2:cradle-cassandra:${cradleVersion}" + def autoValueVersion = '1.10.1' + implementation "com.google.auto.value:auto-value-annotations:$autoValueVersion" + kapt "com.google.auto.value:auto-value:$autoValueVersion" + //this is required to add generated bridge classes for kotlin default constructors + implementation(files("$buildDir/tmp/kapt3/classes/main")) + //FIXME: Add these dependencies as api to grpc-... artifacts implementation "io.grpc:grpc-protobuf" implementation "io.grpc:grpc-core" @@ -224,13 +238,15 @@ dependencies { implementation 'io.prometheus:simpleclient_httpserver' implementation 'io.prometheus:simpleclient_log4j2' - implementation ('com.squareup.okhttp3:okhttp:4.10.0') { - because ('fix vulnerability in transitive dependency ') + implementation('com.squareup.okhttp3:okhttp:4.10.0') { + because('fix vulnerability in transitive dependency ') } - implementation ('io.fabric8:kubernetes-client:6.1.1') { + implementation('io.fabric8:kubernetes-client:6.1.1') { exclude group: 'com.fasterxml.jackson.dataformat', module: 'jackson-dataformat-yaml' } + implementation 'io.github.microutils:kotlin-logging:3.0.0' // The last version bases on kotlin 1.6.0 + testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" testImplementation 'org.mockito.kotlin:mockito-kotlin:4.0.0' testImplementation 'org.jetbrains.kotlin:kotlin-test-junit5' @@ -244,13 +260,13 @@ dependencies { jar { manifest { attributes( - 'Created-By': "${System.getProperty('java.version')} (${System.getProperty('java.vendor')})", - 'Specification-Title': '', - 'Specification-Vendor': 'Exactpro Systems LLC', - 'Implementation-Title': project.archivesBaseName, - 'Implementation-Vendor': 'Exactpro Systems LLC', + 'Created-By': "${System.getProperty('java.version')} (${System.getProperty('java.vendor')})", + 'Specification-Title': '', + 'Specification-Vendor': 'Exactpro Systems LLC', + 'Implementation-Title': project.archivesBaseName, + 'Implementation-Vendor': 'Exactpro Systems LLC', 'Implementation-Vendor-Id': 'com.exactpro', - 'Implementation-Version': project.version + 'Implementation-Version': project.version ) } } @@ -269,8 +285,8 @@ clean { } dependencyCheck { - formats=['SARIF', 'JSON', 'HTML'] - failBuildOnCVSS=5 + formats = ['SARIF', 'JSON', 'HTML'] + failBuildOnCVSS = 5 analyzers { assemblyEnabled = false diff --git a/gradle.properties b/gradle.properties index 7d9fed0c3..b8ef7806b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,6 @@ # # Copyright 2022-2023 Exactpro (Exactpro Systems Limited) +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -12,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.2.0 -description = 'th2 common library (Java)' +release_version=5.3.0 +description='th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j +kapt.include.compile.classpath=false diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/GenericCollectionBuilder.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/GenericCollectionBuilder.java new file mode 100644 index 000000000..04b3a82bb --- /dev/null +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/GenericCollectionBuilder.java @@ -0,0 +1,45 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * This builder is created in order to support @AutoBuilder generation + * The generated builder for types that have generic requires {@code List build()} signature + * That is why there are two builders ({@link GenericCollectionBuilder} and {@link CollectionBuilder}) + * @param collection type + */ +public class GenericCollectionBuilder { + private final List elements = new ArrayList<>(); + + public GenericCollectionBuilder add(T el) { + elements.add(el); + return this; + } + + public GenericCollectionBuilder addAll(Collection els) { + elements.addAll(els); + return this; + } + + public List build() { + return elements; + } +} diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt index d0813db31..a4bc7bc5f 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageFilterUtils.kt @@ -41,17 +41,19 @@ import com.exactpro.th2.common.grpc.ValueFilter.KindCase.SIMPLE_FILTER import com.exactpro.th2.common.value.emptyValueFilter import com.exactpro.th2.common.value.toValueFilter import com.fasterxml.jackson.annotation.JsonProperty -import java.util.* +import java.util.Locale private val DEFAULT_TIME_PRECISION_REGEX = Regex("(\\d[HMS])(?!\$)") fun messageFilter(): MessageFilter.Builder = MessageFilter.newBuilder() -fun rootMessageFilter(messageType: String): RootMessageFilter.Builder = RootMessageFilter.newBuilder().setMessageType(messageType) +fun rootMessageFilter(messageType: String): RootMessageFilter.Builder = + RootMessageFilter.newBuilder().setMessageType(messageType) fun MessageFilter.getField(key: String): ValueFilter? = getFieldsOrDefault(key, null) fun MessageFilter.Builder.getField(key: String): ValueFilter? = getFieldsOrDefault(key, null) -fun MessageFilter.Builder.addField(key: String, value: Any?): MessageFilter.Builder = apply { putFields(key, value?.toValueFilter() ?: emptyValueFilter()) } +fun MessageFilter.Builder.addField(key: String, value: Any?): MessageFilter.Builder = + apply { putFields(key, value?.toValueFilter() ?: emptyValueFilter()) } /** * It accepts vararg with even size and splits it into pairs where the first value of a pair is used as a key while the second is used as a value @@ -62,20 +64,28 @@ fun MessageFilter.Builder.addFields(vararg fields: Any?): MessageFilter.Builder } } -fun MessageFilter.Builder.addFields(fields: Map?): MessageFilter.Builder = apply { fields?.forEach { addField(it.key, it.value) } } +fun MessageFilter.Builder.addFields(fields: Map?): MessageFilter.Builder = + apply { fields?.forEach { addField(it.key, it.value) } } -fun MessageFilter.Builder.copyField(message: MessageFilter.Builder, key: String): MessageFilter.Builder = apply { putFields(key, message.getField(key) ?: emptyValueFilter()) } -fun MessageFilter.Builder.copyField(message: MessageFilter.Builder, vararg key: String): MessageFilter.Builder = apply { key.forEach { putFields(it, message.getField(it) ?: emptyValueFilter()) } } -fun MessageFilter.Builder.copyField(message: MessageFilter, vararg key: String): MessageFilter.Builder = apply { key.forEach { putFields(it, message.getField(it) ?: emptyValueFilter()) } } +fun MessageFilter.Builder.copyField(message: MessageFilter.Builder, key: String): MessageFilter.Builder = + apply { putFields(key, message.getField(key) ?: emptyValueFilter()) } + +fun MessageFilter.Builder.copyField(message: MessageFilter.Builder, vararg key: String): MessageFilter.Builder = + apply { key.forEach { putFields(it, message.getField(it) ?: emptyValueFilter()) } } + +fun MessageFilter.Builder.copyField(message: MessageFilter, vararg key: String): MessageFilter.Builder = + apply { key.forEach { putFields(it, message.getField(it) ?: emptyValueFilter()) } } fun MessageFilter.copy(): MessageFilter.Builder = MessageFilter.newBuilder().putAllFields(fieldsMap) fun MessageFilter.Builder.copy(): MessageFilter.Builder = MessageFilter.newBuilder().putAllFields(fieldsMap) fun RootMessageFilter.toTreeTable(): TreeTable = TreeTableBuilder().apply { - row("message-type", RowBuilder() - .column(MessageTypeColumn(messageType)) - .build()) + row( + "message-type", RowBuilder() + .column(MessageTypeColumn(messageType)) + .build() + ) row("message-filter", messageFilter.toTreeTableEntry()) row("metadata-filter", metadataFilter.toTreeTableEntry()) row("comparison-settings", comparisonSettings.toTreeTableEntry()) @@ -118,26 +128,33 @@ private fun MetadataFilter.toTreeTableEntry(): TreeTableEntry = CollectionBuilde private fun RootComparisonSettings.toTreeTableEntry(): TreeTableEntry = CollectionBuilder().apply { row("ignore-fields", CollectionBuilder().apply { - ignoreFieldsList.forEachIndexed { index, nestedValue -> + ignoreFieldsList.forEachIndexed { index, nestedValue -> val nestedName = index.toString() - row(nestedName, RowBuilder() - .column(IgnoreFieldColumn(nestedValue)) - .build()) + row( + nestedName, RowBuilder() + .column(IgnoreFieldColumn(nestedValue)) + .build() + ) } }.build()) if (hasTimePrecision()) { val timePrecision = timePrecision.toJavaDuration().toString().substring(2) - row("time-precision", RowBuilder() - .column( - IgnoreFieldColumn( - DEFAULT_TIME_PRECISION_REGEX.replace(timePrecision, "$1 ").lowercase(Locale.getDefault()) - )) - .build()) + row( + "time-precision", RowBuilder() + .column( + IgnoreFieldColumn( + DEFAULT_TIME_PRECISION_REGEX.replace(timePrecision, "$1 ").lowercase(Locale.getDefault()) + ) + ) + .build() + ) } if (decimalPrecision.isNotBlank()) { - row("decimal-precision", RowBuilder() - .column(IgnoreFieldColumn(decimalPrecision)) - .build()) + row( + "decimal-precision", RowBuilder() + .column(IgnoreFieldColumn(decimalPrecision)) + .build() + ) } }.build() @@ -153,6 +170,7 @@ private fun SimpleFilter.toTreeTableEntry(): TreeTableEntry = when { filterValueCase == VALUE || operation == EMPTY || operation == NOT_EMPTY -> RowBuilder() .column(MessageFilterTableColumn(value, operation.toString(), key)) .build() + else -> error("Unsupported simple filter value: $filterValueCase") } @@ -163,15 +181,23 @@ private fun ValueFilter.toTreeTableEntry(): TreeTableEntry = when { kindCase == NULL_VALUE -> RowBuilder() .column(MessageFilterTableColumn(if (operation == EQUAL) "IS_NULL" else "IS_NOT_NULL", key)) .build() + kindCase == SIMPLE_FILTER || operation == EMPTY || operation == NOT_EMPTY -> RowBuilder() .column(MessageFilterTableColumn(simpleFilter, operation.toString(), key)) .build() + else -> error("Unsupported ValueFilter value: $kindCase") } private fun SimpleList.toTreeTableEntry(operation: FilterOperation, key: Boolean): TreeTableEntry { return RowBuilder() - .column(MessageFilterTableColumn(simpleValuesList.joinToString(prefix = "[", postfix = "]"), operation.toString(), key)) + .column( + MessageFilterTableColumn( + simpleValuesList.joinToString(prefix = "[", postfix = "]"), + operation.toString(), + key + ) + ) .build() } diff --git a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt index d0fbf1da4..ade4a6d75 100644 --- a/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/message/MessageUtils.kt @@ -80,13 +80,16 @@ import java.math.BigInteger import java.time.Instant import java.time.LocalDateTime import java.time.ZoneOffset -import java.util.* +import java.util.Calendar +import java.util.Date +import java.util.Locale +import java.util.TimeZone typealias FieldValues = Map typealias FieldValueFilters = Map typealias JavaDuration = java.time.Duration -fun message() : Message.Builder = Message.newBuilder() +fun message(): Message.Builder = Message.newBuilder() fun message(messageType: String): Message.Builder = Message.newBuilder().setMetadata(messageType = messageType) fun message(bookName: String, messageType: String, direction: Direction, sessionAlias: String) = Message.newBuilder().setMetadata(bookName, messageType, direction, sessionAlias) @@ -96,8 +99,8 @@ fun Message.getField(fieldName: String): Value? = getFieldsOrDefault(fieldName, operator fun Message.Builder.get(key: String): Value? = getField(key) fun Message.Builder.getField(fieldName: String): Value? = getFieldsOrDefault(fieldName, null) -fun Message.hasField(key: String) : Boolean = fieldsMap.containsKey(key) -fun Message.Builder.hasField(key: String) : Boolean = fieldsMap.containsKey(key) +fun Message.hasField(key: String): Boolean = fieldsMap.containsKey(key) +fun Message.Builder.hasField(key: String): Boolean = fieldsMap.containsKey(key) fun Message.getString(fieldName: String): String? = getField(fieldName)?.getString() fun Message.getInt(fieldName: String): Int? = getField(fieldName)?.getInt() @@ -121,20 +124,47 @@ fun Message.Builder.getList(fieldName: String): List? = getField(fieldNam operator fun Message.Builder.set(key: String, value: Any?): Message.Builder = apply { addField(key, value) } -fun Message.Builder.updateField(key: String, updateFunc: Value.Builder.() -> ValueOrBuilder?): Message.Builder = apply { set(key, updateFunc(getField(key)?.toBuilder() ?: throw NullPointerException("Can not find field with name $key"))) } -fun Message.Builder.updateList(key: String, updateFunc: ListValue.Builder.() -> ListValueOrBuilder) : Message.Builder = apply { updateField(key) { updateList(updateFunc) } } -fun Message.Builder.updateMessage(key: String, updateFunc: Message.Builder.() -> MessageOrBuilder) : Message.Builder = apply { updateField(key) { updateMessage(updateFunc) } } -fun Message.Builder.updateString(key: String, updateFunc: String.() -> String) : Message.Builder = apply { updateField(key) { updateString(updateFunc) } } +fun Message.Builder.updateField(key: String, updateFunc: Value.Builder.() -> ValueOrBuilder?): Message.Builder = apply { + set( + key, + updateFunc(getField(key)?.toBuilder() ?: throw NullPointerException("Can not find field with name $key")) + ) +} + +fun Message.Builder.updateList(key: String, updateFunc: ListValue.Builder.() -> ListValueOrBuilder): Message.Builder = + apply { updateField(key) { updateList(updateFunc) } } + +fun Message.Builder.updateMessage(key: String, updateFunc: Message.Builder.() -> MessageOrBuilder): Message.Builder = + apply { updateField(key) { updateMessage(updateFunc) } } -fun Message.Builder.updateOrAddField(key: String, updateFunc: (Value.Builder?) -> ValueOrBuilder?): Message.Builder = apply { set(key, updateFunc(getField(key)?.toBuilder())) } -fun Message.Builder.updateOrAddList(key: String, updateFunc: (ListValue.Builder?) -> ListValueOrBuilder) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddList(updateFunc) ?: updateFunc(null).toValue() } } -fun Message.Builder.updateOrAddMessage(key: String, updateFunc: (Message.Builder?) -> MessageOrBuilder) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddMessage(updateFunc) ?: updateFunc(null).toValue() } } -fun Message.Builder.updateOrAddString(key: String, updateFunc:(String?) -> String) : Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddString(updateFunc) ?: updateFunc(null).toValue() } } +fun Message.Builder.updateString(key: String, updateFunc: String.() -> String): Message.Builder = + apply { updateField(key) { updateString(updateFunc) } } -fun Message.Builder.addField(key: String, value: Any?): Message.Builder = apply { putFields(key, value?.toValue() ?: nullValue()) } +fun Message.Builder.updateOrAddField(key: String, updateFunc: (Value.Builder?) -> ValueOrBuilder?): Message.Builder = + apply { set(key, updateFunc(getField(key)?.toBuilder())) } + +fun Message.Builder.updateOrAddList( + key: String, + updateFunc: (ListValue.Builder?) -> ListValueOrBuilder +): Message.Builder = apply { updateOrAddField(key) { it?.updateOrAddList(updateFunc) ?: updateFunc(null).toValue() } } + +fun Message.Builder.updateOrAddMessage( + key: String, + updateFunc: (Message.Builder?) -> MessageOrBuilder +): Message.Builder = + apply { updateOrAddField(key) { it?.updateOrAddMessage(updateFunc) ?: updateFunc(null).toValue() } } -fun Message.Builder.copyField(message: Message, key: String) : Message.Builder = apply { if (message.getField(key) != null) putFields(key, message.getField(key)) } -fun Message.Builder.copyField(message: Message.Builder, key: String): Message.Builder = apply { if (message.getField(key) != null) putFields(key, message.getField(key)) } +fun Message.Builder.updateOrAddString(key: String, updateFunc: (String?) -> String): Message.Builder = + apply { updateOrAddField(key) { it?.updateOrAddString(updateFunc) ?: updateFunc(null).toValue() } } + +fun Message.Builder.addField(key: String, value: Any?): Message.Builder = + apply { putFields(key, value?.toValue() ?: nullValue()) } + +fun Message.Builder.copyField(message: Message, key: String): Message.Builder = + apply { if (message.getField(key) != null) putFields(key, message.getField(key)) } + +fun Message.Builder.copyField(message: Message.Builder, key: String): Message.Builder = + apply { if (message.getField(key) != null) putFields(key, message.getField(key)) } /** @@ -146,14 +176,20 @@ fun Message.Builder.addFields(vararg fields: Any?): Message.Builder = apply { } } -fun Message.Builder.addFields(fields: Map?): Message.Builder = apply { fields?.forEach { addField(it.key, it.value?.toValue() ?: nullValue()) } } +fun Message.Builder.addFields(fields: Map?): Message.Builder = + apply { fields?.forEach { addField(it.key, it.value?.toValue() ?: nullValue()) } } -fun Message.Builder.copyFields(message: Message, vararg keys: String) : Message.Builder = apply { keys.forEach { copyField(message, it) } } -fun Message.Builder.copyFields(message: Message.Builder, vararg keys: String) : Message.Builder = apply { keys.forEach { copyField(message, it) } } +fun Message.Builder.copyFields(message: Message, vararg keys: String): Message.Builder = + apply { keys.forEach { copyField(message, it) } } -fun Message.copy(): Message.Builder = Message.newBuilder().setMetadata(metadata).putAllFields(fieldsMap).setParentEventId(parentEventId) +fun Message.Builder.copyFields(message: Message.Builder, vararg keys: String): Message.Builder = + apply { keys.forEach { copyField(message, it) } } -fun Message.Builder.copy(): Message.Builder = Message.newBuilder().setMetadata(metadata).putAllFields(fieldsMap).setParentEventId(parentEventId) +fun Message.copy(): Message.Builder = + Message.newBuilder().setMetadata(metadata).putAllFields(fieldsMap).setParentEventId(parentEventId) + +fun Message.Builder.copy(): Message.Builder = + Message.newBuilder().setMetadata(metadata).putAllFields(fieldsMap).setParentEventId(parentEventId) fun Message.Builder.setMetadata( bookName: String? = null, @@ -204,9 +240,9 @@ operator fun MessageGroup.Builder.plusAssign(rawMessage: RawMessage.Builder) { fun Instant.toTimestamp(): Timestamp = Timestamp.newBuilder().setSeconds(epochSecond).setNanos(nano).build() fun Date.toTimestamp(): Timestamp = toInstant().toTimestamp() -fun LocalDateTime.toTimestamp(zone: ZoneOffset) : Timestamp = toInstant(zone).toTimestamp() -fun LocalDateTime.toTimestamp() : Timestamp = toTimestamp(ZoneOffset.of(TimeZone.getDefault().id)) -fun Calendar.toTimestamp() : Timestamp = toInstant().toTimestamp() +fun LocalDateTime.toTimestamp(zone: ZoneOffset): Timestamp = toInstant(zone).toTimestamp() +fun LocalDateTime.toTimestamp(): Timestamp = toTimestamp(ZoneOffset.of(TimeZone.getDefault().id)) +fun Calendar.toTimestamp(): Timestamp = toInstant().toTimestamp() fun Duration.toJavaDuration(): JavaDuration = JavaDuration.ofSeconds(seconds, nanos.toLong()) fun JavaDuration.toProtoDuration(): Duration = Duration.newBuilder().setSeconds(seconds).setNanos(nano).build() @@ -277,17 +313,18 @@ fun Value.toValueFilter(isKey: Boolean): ValueFilter = when (val source = this) fun FieldValues.toFieldValueFilters(keyFields: List = listOf()): FieldValueFilters = mapValues { (name, value) -> value.toValueFilter(name in keyFields) } -fun Message.toMessageFilter(keyFields: List = listOf(), failUnexpected: FailUnexpected = NO): MessageFilter = when (val source = this) { - Message.getDefaultInstance() -> MessageFilter.getDefaultInstance() - else -> MessageFilter.newBuilder().apply { - putAllFields(source.fieldsMap.toFieldValueFilters(keyFields)) - if (failUnexpected.number != 0) { - comparisonSettingsBuilder.apply { - this.failUnexpected = failUnexpected +fun Message.toMessageFilter(keyFields: List = listOf(), failUnexpected: FailUnexpected = NO): MessageFilter = + when (val source = this) { + Message.getDefaultInstance() -> MessageFilter.getDefaultInstance() + else -> MessageFilter.newBuilder().apply { + putAllFields(source.fieldsMap.toFieldValueFilters(keyFields)) + if (failUnexpected.number != 0) { + comparisonSettingsBuilder.apply { + this.failUnexpected = failUnexpected + } } - } - }.build() -} + }.build() + } fun ListValue.toListValueFilter(): ListValueFilter { return if (ListValue.getDefaultInstance() == this) { @@ -428,7 +465,8 @@ val AnyMessage.logId: String else -> error("Cannot get log id from $kindCase message: ${toJson()}") } -fun getSessionAliasAndDirection(messageID: MessageID): Array = arrayOf(messageID.connectionId.sessionAlias, messageID.direction.name) +fun getSessionAliasAndDirection(messageID: MessageID): Array = + arrayOf(messageID.connectionId.sessionAlias, messageID.direction.name) fun getSessionAliasAndDirection(anyMessage: AnyMessage): Array = when { anyMessage.hasMessage() -> getSessionAliasAndDirection(anyMessage.message.metadata.id) @@ -457,14 +495,19 @@ fun getDebugString(className: String, ids: List): String { } @JvmOverloads -fun com.google.protobuf.MessageOrBuilder.toJson(short: Boolean = true): String = JsonFormat.printer().includingDefaultValueFields().let { - (if (short) it.omittingInsignificantWhitespace() else it).print(this) -} +fun com.google.protobuf.MessageOrBuilder.toJson(short: Boolean = true): String = + JsonFormat.printer().includingDefaultValueFields().let { + (if (short) it.omittingInsignificantWhitespace() else it).print(this) + } -fun T.fromJson(json: String) : T = apply { +fun T.fromJson(json: String): T = apply { JsonFormat.parser().ignoringUnknownFields().merge(json, this) } +@Deprecated( + "Moved to common-utils-j/com.exactpro.th2.common.utils.message.MessageUtils", + replaceWith = ReplaceWith("toTreeTable()", "com.exactpro.th2.common.utils.message.toTreeTable") +) fun Message.toTreeTable(): TreeTable = TreeTableBuilder().apply { for ((key, value) in fieldsMap) { row(key, value.toTreeTableEntry()) @@ -474,24 +517,37 @@ fun Message.toTreeTable(): TreeTable = TreeTableBuilder().apply { val MessageIDOrBuilder.isValid: Boolean get() = bookName.isNotBlank() && hasConnectionId() && connectionId.sessionAlias.isNotBlank() - && hasTimestamp() && timestamp.seconds > 0 && timestamp.nanos > 0 + && hasTimestamp() && (timestamp.seconds > 0 || timestamp.nanos > 0) && sequence > 0 +@Deprecated( + "Moved to common-utils-j/com.exactpro.th2.common.utils.message.MessageUtils", + replaceWith = ReplaceWith("toTreeTable()", "com.exactpro.th2.common.utils.message.toTreeTableEntry") +) private fun Value.toTreeTableEntry(): TreeTableEntry = when { hasMessageValue() -> CollectionBuilder().apply { for ((key, value) in messageValue.fieldsMap) { row(key, value.toTreeTableEntry()) } }.build() + hasListValue() -> CollectionBuilder().apply { listValue.valuesList.forEachIndexed { index, nestedValue -> val nestedName = index.toString() row(nestedName, nestedValue.toTreeTableEntry()) } }.build() + else -> RowBuilder() .column(MessageTableColumn(simpleValue)) .build() } -internal data class MessageTableColumn(val fieldValue: String) : IColumn +@Deprecated( + "Moved to common-utils-j/com.exactpro.th2.common.utils.message.MessageUtils", + replaceWith = ReplaceWith( + "MessageTableColumn(fieldValue)", + "com.exactpro.th2.common.utils.message.MessageTableColumn" + ) +) +data class MessageTableColumn(val fieldValue: String) : IColumn diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Cleanable.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Cleanable.kt index df55289b5..673c53681 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Cleanable.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Cleanable.kt @@ -21,10 +21,4 @@ interface Cleanable { * Deep clean */ fun clean() - - /** - * Cleans only current layer of th2 entity include collections. - * This method can be useful when you need th2 entity builder, but you are going to operate th2 entity separately - */ - fun softClean() = clean() } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt index 69dc3f610..0e43f0c65 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt @@ -16,15 +16,15 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.dataformat.cbor.databind.CBORMapper +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.kotlin.jacksonTypeRef +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBufInputStream import io.netty.buffer.ByteBufOutputStream import io.netty.buffer.ByteBufUtil -import java.io.InputStream import java.io.OutputStream import java.nio.charset.Charset import java.time.Instant @@ -53,7 +53,7 @@ enum class ValueType(val codec: ValueCodec<*>) { RAW_MESSAGE(RawMessageCodec), RAW_MESSAGE_BODY(RawMessageBodyCodec), PARSED_MESSAGE(ParsedMessageCodec), - PARSED_MESSAGE_BODY(ParsedMessageBodyCodec), + PARSED_MESSAGE_BODY(ParsedMessageRawBodyCodec), MESSAGE_GROUP(MessageGroupCodec), MESSAGE_LIST(MessageListCodec), GROUP_BATCH(GroupBatchCodec), @@ -133,14 +133,14 @@ abstract class IntCodec(type: UByte) : AbstractCodec(type) { } } -abstract class ListCodec(type: UByte, private val elementCodec: ValueCodec) : AbstractCodec>(type) { +abstract class ListCodec(type: UByte, private val elementCodec: ValueCodec) : AbstractCodec>(type) { override fun read(buffer: ByteBuf): MutableList = mutableListOf().also { list -> while (buffer.isReadable) { list += elementCodec.decode(buffer) } } - override fun write(buffer: ByteBuf, value: MutableList) { + override fun write(buffer: ByteBuf, value: List) { value.forEach { elementCodec.encode(it, buffer) } } } @@ -149,14 +149,14 @@ abstract class MapCodec( type: UByte, private val keyCodec: ValueCodec, private val valueCodec: ValueCodec, -) : AbstractCodec>(type) { +) : AbstractCodec>(type) { override fun read(buffer: ByteBuf): MutableMap = hashMapOf().apply { while (buffer.isReadable) { this[keyCodec.decode(buffer)] = valueCodec.decode(buffer) } } - override fun write(buffer: ByteBuf, value: MutableMap): Unit = value.forEach { (key, value) -> + override fun write(buffer: ByteBuf, value: Map): Unit = value.forEach { (key, value) -> keyCodec.encode(key, buffer) valueCodec.encode(value, buffer) } @@ -178,20 +178,6 @@ abstract class InstantCodec(type: UByte) : AbstractCodec(type) { } } -open class CborCodec(type: UByte, private val typeReference: TypeReference) : AbstractCodec(type) { - override fun read(buffer: ByteBuf): T = ByteBufInputStream(buffer).use { - return MAPPER.readValue(it as InputStream, typeReference) - } - - override fun write(buffer: ByteBuf, value: T) = ByteBufOutputStream(buffer).use { - MAPPER.writeValue(it as OutputStream, value) - } - - companion object { - private val MAPPER = CBORMapper().registerModule(JavaTimeModule()) - } -} - // FIXME: think about checking that type is unique object LongTypeCodec : LongCodec(1u) @@ -200,18 +186,18 @@ object StringTypeCodec : StringCodec(2u) object IntTypeCodec : IntCodec(3u) object MessageIdCodec : AbstractCodec(10u) { - override fun read(buffer: ByteBuf): MessageId = MessageId().apply { + override fun read(buffer: ByteBuf): MessageId = MessageId.builder().apply { buffer.forEachValue { codec -> when (codec) { - is SessionAliasCodec -> sessionAlias = codec.decode(buffer) - is DirectionCodec -> direction = codec.decode(buffer) - is SequenceCodec -> sequence = codec.decode(buffer) - is SubsequenceCodec -> subsequence = codec.decode(buffer) - is TimestampCodec -> timestamp = codec.decode(buffer) + is SessionAliasCodec -> setSessionAlias(codec.decode(buffer)) + is DirectionCodec -> setDirection(codec.decode(buffer)) + is SequenceCodec -> setSequence(codec.decode(buffer)) + is SubsequenceCodec -> setSubsequence(codec.decode(buffer)) + is TimestampCodec -> setTimestamp(codec.decode(buffer)) else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") } } - } + }.build() override fun write(buffer: ByteBuf, value: MessageId) { SessionAliasCodec.encode(value.sessionAlias, buffer) @@ -254,21 +240,17 @@ object ScopeCodec : StringCodec(15u) object EventIdCodec : AbstractCodec(16u) { override fun read(buffer: ByteBuf): EventId { - var id = "" - var book = "" - var scope = "" - var timestamp: Instant = Instant.EPOCH - - buffer.forEachValue { codec -> - when (codec) { - is IdCodec -> id = codec.decode(buffer) - is BookCodec -> book = codec.decode(buffer) - is ScopeCodec -> scope = codec.decode(buffer) - is TimestampCodec -> timestamp = codec.decode(buffer) - else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + return EventId.builder().apply { + buffer.forEachValue { codec -> + when (codec) { + is IdCodec -> setId(codec.decode(buffer)) + is BookCodec -> setBook(codec.decode(buffer)) + is ScopeCodec -> setScope(codec.decode(buffer)) + is TimestampCodec -> setTimestamp(codec.decode(buffer)) + else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") + } } - } - return EventId(id, book, scope, timestamp) + }.build() } override fun write(buffer: ByteBuf, value: EventId) { @@ -280,18 +262,18 @@ object EventIdCodec : AbstractCodec(16u) { } object RawMessageCodec : AbstractCodec(20u) { - override fun read(buffer: ByteBuf): RawMessage = RawMessage().apply { + override fun read(buffer: ByteBuf): RawMessage = RawMessage.builder().apply { buffer.forEachValue { codec -> when (codec) { - is MessageIdCodec -> id = codec.decode(buffer) - is EventIdCodec -> eventId = codec.decode(buffer) - is MetadataCodec -> metadata = codec.decode(buffer) - is ProtocolCodec -> protocol = codec.decode(buffer) - is RawMessageBodyCodec -> body = codec.decode(buffer) + is MessageIdCodec -> setId(codec.decode(buffer)) + is EventIdCodec -> setEventId(codec.decode(buffer)) + is MetadataCodec -> setMetadata(codec.decode(buffer)) + is ProtocolCodec -> setProtocol(codec.decode(buffer)) + is RawMessageBodyCodec -> setBody(codec.decode(buffer)) else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") } } - } + }.build() override fun write(buffer: ByteBuf, value: RawMessage) { MessageIdCodec.encode(value.id, buffer) @@ -305,19 +287,21 @@ object RawMessageCodec : AbstractCodec(20u) { object RawMessageBodyCodec : ByteBufCodec(21u) object ParsedMessageCodec : AbstractCodec(30u) { - override fun read(buffer: ByteBuf): ParsedMessage = ParsedMessage().apply { + override fun read(buffer: ByteBuf): ParsedMessage = ParsedMessage.builder { buf -> + ByteBufInputStream(buf).use { MAPPER.readValue(it) } + }.apply { buffer.forEachValue { codec -> when (codec) { - is MessageIdCodec -> id = codec.decode(buffer) - is EventIdCodec -> eventId = codec.decode(buffer) - is MetadataCodec -> metadata = codec.decode(buffer) - is ProtocolCodec -> protocol = codec.decode(buffer) - is MessageTypeCodec -> type = codec.decode(buffer) - is ParsedMessageBodyCodec -> body = codec.decode(buffer) + is MessageIdCodec -> setId(codec.decode(buffer)) + is EventIdCodec -> setEventId(codec.decode(buffer)) + is MetadataCodec -> setMetadata(codec.decode(buffer)) + is ProtocolCodec -> setProtocol(codec.decode(buffer)) + is MessageTypeCodec -> setType(codec.decode(buffer)) + is ParsedMessageRawBodyCodec -> setRawBody(codec.decode(buffer)) else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") } } - } + }.build() override fun write(buffer: ByteBuf, value: ParsedMessage) { MessageIdCodec.encode(value.id, buffer) @@ -325,28 +309,39 @@ object ParsedMessageCodec : AbstractCodec(30u) { MetadataCodec.encode(value.metadata, buffer) ProtocolCodec.encode(value.protocol, buffer) MessageTypeCodec.encode(value.type, buffer) - ParsedMessageBodyCodec.encode(value.body, buffer) + if (!value.isBodyInRaw) { + // Update raw body because the body was changed + ByteBufOutputStream(value.rawBody.clear()).use { + MAPPER.writeValue(it as OutputStream, value.body) + } + value.rawBody.resetReaderIndex() + } + ParsedMessageRawBodyCodec.encode(value.rawBody, buffer) } + + @JvmField + val MAPPER: ObjectMapper = jacksonObjectMapper().registerModule(JavaTimeModule()) + .setSerializationInclusion(JsonInclude.Include.ALWAYS) } -object ParsedMessageBodyCodec : CborCodec>(31u, jacksonTypeRef()) +object ParsedMessageRawBodyCodec : ByteBufCodec(31u) object MessageGroupCodec : AbstractCodec(40u) { - override fun read(buffer: ByteBuf): MessageGroup = MessageGroup().apply { + override fun read(buffer: ByteBuf): MessageGroup = MessageGroup.builder().apply { buffer.forEachValue { codec -> when (codec) { - is MessageListCodec -> messages = codec.decode(buffer) + is MessageListCodec -> setMessages(codec.decode(buffer)) else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") } } - } + }.build() override fun write(buffer: ByteBuf, value: MessageGroup) { MessageListCodec.encode(value.messages, buffer) } } -object MessageListCodec : AbstractCodec>>(41u) { +object MessageListCodec : AbstractCodec>>(41u) { override fun read(buffer: ByteBuf): MutableList> = mutableListOf>().apply { buffer.forEachValue { codec -> when (codec) { @@ -357,7 +352,7 @@ object MessageListCodec : AbstractCodec>>(41u) { } } - override fun write(buffer: ByteBuf, value: MutableList>): Unit = value.forEach { message -> + override fun write(buffer: ByteBuf, value: List>): Unit = value.forEach { message -> when (message) { is RawMessage -> RawMessageCodec.encode(message, buffer) is ParsedMessage -> ParsedMessageCodec.encode(message, buffer) @@ -367,16 +362,16 @@ object MessageListCodec : AbstractCodec>>(41u) { } object GroupBatchCodec : AbstractCodec(50u) { - override fun read(buffer: ByteBuf): GroupBatch = GroupBatch().apply { + override fun read(buffer: ByteBuf): GroupBatch = GroupBatch.builder().apply { buffer.forEachValue { codec -> when (codec) { - is BookCodec -> book = codec.decode(buffer) - is SessionGroupCodec -> sessionGroup = codec.decode(buffer) - is GroupListCodec -> groups = codec.decode(buffer) + is BookCodec -> setBook(codec.decode(buffer)) + is SessionGroupCodec -> setSessionGroup(codec.decode(buffer)) + is GroupListCodec -> setGroups(codec.decode(buffer)) else -> println("Skipping unexpected type ${codec.type} value: ${codec.decode(buffer)}") } } - } + }.build() override fun write(buffer: ByteBuf, value: GroupBatch) { BookCodec.encode(value.book, buffer) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt index 51a5e2aa7..56508e31b 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt @@ -16,28 +16,31 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport +import com.google.auto.value.AutoBuilder import java.time.Instant data class EventId( - var id: String = "", - var book: String = "", - var scope: String = "", - var timestamp: Instant = Instant.EPOCH, -) : Cleanable { + val id: String, + val book: String, + val scope: String, + val timestamp: Instant, +) { + @AutoBuilder + interface Builder { + val id: String + val book: String + val scope: String + val timestamp: Instant - override fun clean() { - check(this !== DEFAULT_INSTANCE) { - "Object can be cleaned because it is default instance" - } - id= "" - book = "" - scope = "" - timestamp = Instant.EPOCH + fun setId(id: String): Builder + fun setBook(book: String): Builder + fun setScope(scope: String): Builder + fun setTimestamp(timestamp: Instant): Builder + fun build(): EventId } companion object { - val DEFAULT_INSTANCE: EventId = EventId() // FIXME: do smth about its mutability @JvmStatic - fun newMutable() = EventId() + fun builder(): Builder = AutoBuilder_EventId_Builder() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt index 3c9c06ae0..74e98c6f3 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt @@ -16,26 +16,32 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import java.util.* +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.CollectionBuilder +import com.google.auto.value.AutoBuilder data class GroupBatch( - var book: String = "", - var sessionGroup: String = "", - var groups: MutableList = DEFAULT_GROUPS, -) : Cleanable { - override fun clean() { - check(groups !== DEFAULT_GROUPS) { - "Object can be cleaned because 'groups' is immutable" + val book: String, + val sessionGroup: String, + val groups: List = emptyList(), +) { + @AutoBuilder + interface Builder { + val book: String + val sessionGroup: String + + fun setBook(book: String): Builder + fun setSessionGroup(sessionGroup: String): Builder + fun groupsBuilder(): CollectionBuilder + fun addGroup(group: MessageGroup): Builder = apply { + groupsBuilder().add(group) } - groups.clear() + fun setGroups(groups: List): Builder + fun build(): GroupBatch } companion object { - val DEFAULT_GROUPS: MutableList = Collections.emptyList() @JvmStatic - fun newMutable() = GroupBatch( - groups = mutableListOf() - ) + fun builder(): Builder = AutoBuilder_GroupBatch_Builder() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt index cd452f010..69188708e 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt @@ -16,18 +16,30 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import java.util.* +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.MapBuilder -interface Message : Cleanable { +interface Message { /** The id is not mutable by default */ - var id: MessageId - var eventId: EventId? + val id: MessageId + val eventId: EventId? + /** The metadata is not mutable by default */ - var metadata: MutableMap - var protocol: String - var body: T + val metadata: Map + val protocol: String + val body: T + + interface Builder> { + val protocol: String + val eventId: EventId? + + fun setId(id: MessageId): T + fun idBuilder(): MessageId.Builder + fun setEventId(eventId: EventId): T + fun setProtocol(protocol: String): T + fun setMetadata(metadata: Map): T + fun metadataBuilder(): MapBuilder - companion object { - val DEFAULT_METADATA: MutableMap = Collections.emptyMap() + fun addMetadataProperty(key: String, value: String): T + fun build(): Message<*> } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt index 7d096cf91..3756e548b 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt @@ -16,21 +16,47 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import java.util.* +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.GenericCollectionBuilder +import com.google.auto.value.AutoBuilder + data class MessageGroup( - var messages: MutableList> = DEFAULT_MESSAGES, // FIXME: message can have incompatible book and group -) : Cleanable { - override fun clean() { - check(messages !== DEFAULT_MESSAGES) { - "Object can be cleaned because 'messages' is immutable" + val messages: List> = emptyList(), // FIXME: message can have incompatible book and group +) { + // We cannot use AutoBuilder here because of the different method signatures in builder when a generic type is used + // + @AutoBuilder + interface Builder { + fun messagesBuilder(): GenericCollectionBuilder> + fun addMessage(message: Message<*>): Builder = apply { + messagesBuilder().add(message) + } + fun setMessages(message: List>): Builder + fun build(): MessageGroup + } + + class CollectionBuilder { + private val elements: MutableList> = mutableListOf() + + fun add(el: Message<*>): CollectionBuilder = apply { + elements += el } - messages.clear() + + fun addAll(vararg els: Message<*>): CollectionBuilder = apply { + for (el in els) { + add(el) + } + } + + fun addAll(elements: Collection>): CollectionBuilder = apply { + this.elements.addAll(elements) + } + + fun build(): List> = elements } companion object { - val DEFAULT_MESSAGES: MutableList> = Collections.emptyList() @JvmStatic - fun newMutable() = MessageGroup(mutableListOf()) + fun builder(): Builder = AutoBuilder_MessageGroup_Builder() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt index 2dc61d2d7..711d16446 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt @@ -16,38 +16,45 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.CollectionBuilder +import com.google.auto.value.AutoBuilder import java.time.Instant -import java.util.* data class MessageId( - var sessionAlias: String = "", - var direction: Direction = Direction.INCOMING, - var sequence: Long = 0, + val sessionAlias: String, + val direction: Direction, + val sequence: Long, + val timestamp: Instant, /** The subsequence is not mutable by default */ - var subsequence: MutableList = DEFAULT_SUBSEQUENCE, - var timestamp: Instant = Instant.EPOCH, -) : Cleanable { - override fun clean() { - check(this !== DEFAULT_INSTANCE) { - "Object can be cleaned because it is default instance" - } - check(subsequence !== DEFAULT_SUBSEQUENCE) { - "Object can be cleaned because 'subsequence' is immutable" + val subsequence: List = emptyList(), +) { + @AutoBuilder + interface Builder { + val sessionAlias: String + val direction: Direction + val sequence: Long + val timestamp: Instant + + fun setSessionAlias(sessionAlias: String): Builder + fun setDirection(direction: Direction): Builder + fun setSequence(sequence: Long): Builder + fun setTimestamp(timestamp: Instant): Builder + fun subsequenceBuilder(): CollectionBuilder + fun addSubsequence(subsequnce: Int): Builder = apply { + subsequenceBuilder().add(subsequnce) } - sessionAlias = "" - direction = Direction.INCOMING - sequence = 0 - subsequence.clear() - timestamp = Instant.EPOCH + fun setSubsequence(subsequence: List): Builder + fun build(): MessageId } + fun toBuilder(): MessageId.Builder = AutoBuilder_MessageId_Builder(this) + companion object { - val DEFAULT_SUBSEQUENCE: MutableList = Collections.emptyList() - val DEFAULT_INSTANCE: MessageId = MessageId() // FIXME: do smth about its mutability @JvmStatic - fun newMutable() = MessageId( - subsequence = mutableListOf() - ) + val DEFAULT: MessageId = MessageId("", Direction.OUTGOING, 0, Instant.EPOCH) + + @JvmStatic + fun builder(): Builder = AutoBuilder_MessageId_Builder() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt index 5c39dd167..4bb2d5274 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt @@ -16,64 +16,276 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import java.util.* - -data class ParsedMessage( - override var id: MessageId = MessageId.DEFAULT_INSTANCE, - override var eventId: EventId? = null, - override var metadata: MutableMap = Message.DEFAULT_METADATA, - override var protocol: String = "", - var type: String = "", - /** The body is not mutable by default */ - override var body: MutableMap = DEFAULT_BODY, // FIXME: should be lazy deserializing -) : Message> { - override fun clean() { - check(id !== MessageId.DEFAULT_INSTANCE) { - "Object can be cleaned because 'id' is default instance" +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.MapBuilder +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled +import java.util.Collections + +class ParsedMessage private constructor( + override val id: MessageId = MessageId.DEFAULT, + override val eventId: EventId? = null, + val type: String, + override val metadata: Map = emptyMap(), + override val protocol: String = "", + val rawBody: ByteBuf = Unpooled.buffer(), + private val bodySupplier: (ByteBuf) -> Map = DEFAULT_BODY_SUPPLIER, + body: Map = DEFAULT_BODY, +) : Message> { + constructor( + id: MessageId = MessageId.DEFAULT, + eventId: EventId? = null, + type: String, + metadata: Map = emptyMap(), + protocol: String = "", + rawBody: ByteBuf = Unpooled.buffer(), + bodySupplier: (ByteBuf) -> Map = DEFAULT_BODY_SUPPLIER, + ) : this( + id = id, + eventId = eventId, + type = type, + metadata = metadata, + protocol = protocol, + rawBody = rawBody, + bodySupplier = bodySupplier, + body = DEFAULT_BODY, + ) + + constructor( + id: MessageId = MessageId.DEFAULT, + eventId: EventId? = null, + type: String, + metadata: Map = emptyMap(), + protocol: String = "", + body: Map, + ) : this( + id = id, + eventId = eventId, + type = type, + metadata = metadata, + protocol = protocol, + rawBody = Unpooled.buffer(), + bodySupplier = DEFAULT_BODY_SUPPLIER, + body = body, + ) + + /** + * Is set to `true` if the [body] is deserialized from the [rawBody]. + * If the [body] is set directly returns `false` + */ + val isBodyInRaw: Boolean = body === DEFAULT_BODY + + override val body: Map by lazy { + if (body === DEFAULT_BODY) { + bodySupplier.invoke(rawBody).apply { + rawBody.resetReaderIndex() + } + } else { + body + } + } + + + interface Builder> : Message.Builder { + val type: String + + fun setType(type: String): T + override fun build(): ParsedMessage + } + + interface FromRawBuilder : Builder { + fun setRawBody(rawBody: ByteBuf): FromRawBuilder + } + + interface FromMapBuilder : Builder { + fun setBody(body: Map): FromMapBuilder + fun bodyBuilder(): MapBuilder + fun addField(name: String, value: Any?): FromMapBuilder + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as ParsedMessage + + if (id != other.id) return false + if (eventId != other.eventId) return false + if (type != other.type) return false + if (metadata != other.metadata) return false + if (protocol != other.protocol) return false + return rawBody == other.rawBody + } + + override fun hashCode(): Int { + var result = id.hashCode() + result = 31 * result + (eventId?.hashCode() ?: 0) + result = 31 * result + type.hashCode() + result = 31 * result + metadata.hashCode() + result = 31 * result + protocol.hashCode() + result = 31 * result + rawBody.hashCode() + return result + } + + override fun toString(): String { + return "ParsedMessage(id=$id, " + + "eventId=$eventId, " + + "type='$type', " + + "metadata=$metadata, " + + "protocol='$protocol', " + + "rawBody=${rawBody.toString(Charsets.UTF_8)}, " + + "body=${ + if (isBodyInRaw) { + "!checkRawBody" + } else { + body.toString() + } + })" + } + + + companion object { + val EMPTY = ParsedMessage(type = "", body = emptyMap()) + + /** + * We want to be able to identify the default body by reference. + * So, that is why we use unmodifiableMap with emptyMap + * Otherwise, we won't be able to identify it + */ + private val DEFAULT_BODY: Map = Collections.unmodifiableMap(emptyMap()) + + val DEFAULT_BODY_SUPPLIER: (ByteBuf) -> Map = { emptyMap() } + + @JvmStatic + fun builder(bodySupplier: (ByteBuf) -> Map): FromRawBuilder = FromRawBuilderImpl(bodySupplier) + + @JvmStatic + fun builder(): FromMapBuilder = FromMapBuilderImpl() + } +} + +@Suppress("PropertyName") +private sealed class BaseParsedBuilder> : ParsedMessage.Builder { + protected var idBuilder: MessageId.Builder? = null + protected var id: MessageId? = MessageId.DEFAULT + final override var eventId: EventId? = null + private set + protected var _protocol: String? = null + protected var _type: String? = null + protected var metadataBuilder: MapBuilder? = null + protected var metadata: Map? = emptyMap() + + override val protocol: String + get() = this._protocol ?: "" + override val type: String + get() = requireNotNull(this._type) { + "Property \"type\" has not been set" } - check(metadata !== Message.DEFAULT_METADATA) { - "Object can be cleaned because 'metadata' is immutable" + + override fun setId(id: MessageId): T = self { + require(idBuilder == null) { + "cannot set id after calling idBuilder()" } - check(body !== DEFAULT_BODY) { - "Object can be cleaned because 'body' is immutable" + this.id = id + } + + override fun idBuilder(): MessageId.Builder { + if (idBuilder == null) { + idBuilder = id?.toBuilder()?.also { + id = null + } ?: MessageId.builder() } + return requireNotNull(idBuilder) { "idBuilder is null" } + } + + override fun setEventId(eventId: EventId): T = self { + this.eventId = eventId + } + + override fun setProtocol(protocol: String): T = self { + this._protocol = protocol + } + + override fun setMetadata(metadata: Map): T = self { + require(metadataBuilder == null) { + "cannot set metadata after calling metadataBuilder()" + } + this.metadata = metadata + } - id.clean() - eventId = null - metadata.clear() - protocol = "" - type = "" - body.clear() + override fun setType(type: String): T = self { + this._type = type } - override fun softClean() { - check(metadata !== Message.DEFAULT_METADATA) { - "Object can be cleaned because 'metadata' is immutable" + override fun metadataBuilder(): MapBuilder { + if (metadataBuilder == null) { + metadataBuilder = metadata?.let { + metadata = null + MapBuilder().putAll(it) + } ?: MapBuilder() } - check(body !== DEFAULT_BODY) { - "Object can be cleaned because 'body' is immutable" + return requireNotNull(metadataBuilder) { "metadataBuilder is null" } + } + + override fun addMetadataProperty(key: String, value: String): T = self { + metadataBuilder().put(key, value) + } + + @Suppress("UNCHECKED_CAST") + private inline fun self(block: BaseParsedBuilder.() -> Unit): T { + block() + return this as T + } +} + +private class FromRawBuilderImpl( + private val bodySupplier: (ByteBuf) -> Map, +) : BaseParsedBuilder(), ParsedMessage.FromRawBuilder { + private var rawBody: ByteBuf? = null + override fun setRawBody(rawBody: ByteBuf): ParsedMessage.FromRawBuilder = apply { + this.rawBody = rawBody + } + + override fun build(): ParsedMessage = ParsedMessage( + id = id ?: idBuilder?.build() ?: error("missing id"), + eventId = eventId, + type = _type ?: error("missing type"), + metadata = metadata ?: metadataBuilder?.build() ?: emptyMap(), + protocol = _protocol ?: "", + rawBody = rawBody ?: error("missing raw body"), + bodySupplier = bodySupplier, + ) +} + +private class FromMapBuilderImpl : BaseParsedBuilder(), ParsedMessage.FromMapBuilder { + private var body: Map? = null + private var bodyBuilder: MapBuilder? = null + override fun setBody(body: Map): ParsedMessage.FromMapBuilder = apply { + require(bodyBuilder == null) { + "cannot set body after calling bodyBuilder()" } + this.body = body + } - id = MessageId.DEFAULT_INSTANCE - eventId = null - metadata.clear() - protocol = "" - type = "" - body.clear() + override fun bodyBuilder(): MapBuilder { + if (bodyBuilder == null) { + bodyBuilder = body?.let { + body = null + MapBuilder().putAll(it) + } ?: MapBuilder() + } + return requireNotNull(bodyBuilder) { "bodyBuilder is null" } } - companion object { - val DEFAULT_BODY: MutableMap = Collections.emptyMap() - @JvmStatic - fun newMutable() = ParsedMessage( - id = MessageId.newMutable(), - metadata = hashMapOf(), - body = hashMapOf() - ) - @JvmStatic - fun newSoftMutable() = ParsedMessage( - metadata = hashMapOf(), - body = hashMapOf() - ) + override fun addField(name: String, value: Any?): ParsedMessage.FromMapBuilder = apply { + bodyBuilder().put(name, value) } + + override fun build(): ParsedMessage = ParsedMessage( + id = id ?: idBuilder?.build() ?: error("missing id"), + eventId = eventId, + type = _type ?: error("missing type"), + metadata = metadata ?: metadataBuilder?.build() ?: emptyMap(), + protocol = _protocol ?: "", + body = body ?: bodyBuilder?.build() ?: error("missing body"), + ) } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt index a00b2624c..5bcd7b787 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt @@ -16,57 +16,36 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport +import com.google.auto.value.AutoBuilder import io.netty.buffer.ByteBuf import io.netty.buffer.Unpooled data class RawMessage( - override var id: MessageId = MessageId.DEFAULT_INSTANCE, - override var eventId: EventId? = null, - override var metadata: MutableMap = Message.DEFAULT_METADATA, - override var protocol: String = "", + override val id: MessageId = MessageId.DEFAULT, + override val eventId: EventId? = null, + override val metadata: Map = emptyMap(), + override val protocol: String = "", /** The body is not mutable by default */ - override var body: ByteBuf = Unpooled.EMPTY_BUFFER, + override val body: ByteBuf = Unpooled.EMPTY_BUFFER, ) : Message { - override fun clean() { - check(id !== MessageId.DEFAULT_INSTANCE) { - "Object can be cleaned because 'id' is default instance" - } - check(metadata !== Message.DEFAULT_METADATA) { - "Object can be cleaned because 'metadata' is immutable" - } - check(body !== Unpooled.EMPTY_BUFFER) { - "Object can be cleaned because 'body' is immutable" - } + @AutoBuilder + interface Builder : Message.Builder { + val body: ByteBuf - id.clean() - eventId = null - metadata.clear() - protocol = "" - body.clear() - } + fun setBody(body: ByteBuf): Builder + fun setBody(data: ByteArray): Builder = setBody(Unpooled.wrappedBuffer(data)) - override fun softClean() { - check(metadata !== Message.DEFAULT_METADATA) { - "Object can be cleaned because 'metadata' is immutable" + override fun addMetadataProperty(key: String, value: String): Builder = this.apply { + metadataBuilder().put(key, value) } - id = MessageId.DEFAULT_INSTANCE - eventId = null - metadata.clear() - protocol = "" - body = Unpooled.EMPTY_BUFFER + override fun build(): RawMessage } companion object { + val EMPTY = RawMessage() + @JvmStatic - fun newMutable() = RawMessage( - id = MessageId.newMutable(), - metadata = hashMapOf(), - body = Unpooled.buffer() - ) - @JvmStatic - fun newSoftMutable() = RawMessage( - metadata = hashMapOf() - ) + fun builder(): Builder = AutoBuilder_RawMessage_Builder() } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt index 206b5e922..1881d62c6 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt @@ -18,9 +18,6 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.exactpro.th2.common.grpc.Direction.FIRST import com.exactpro.th2.common.grpc.Direction.SECOND -import com.exactpro.th2.common.grpc.EventID -import com.exactpro.th2.common.grpc.MessageID -import com.exactpro.th2.common.message.toTimestamp import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.BOOK_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.DIRECTION_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY @@ -35,20 +32,6 @@ import io.netty.buffer.ByteBuf import io.netty.buffer.Unpooled import com.exactpro.th2.common.grpc.Direction as ProtoDirection -fun GroupBatch.toByteArray(): ByteArray = Unpooled.buffer().run { - GroupBatchCodec.encode(this@toByteArray, this@run) - ByteArray(readableBytes()).apply(::readBytes) -} - -fun ByteBuf.toByteArray(): ByteArray = ByteArray(readableBytes()) - .apply(::readBytes).also { resetReaderIndex() } - -val Direction.proto: ProtoDirection - get() = when (this) { - INCOMING -> FIRST - OUTGOING -> SECOND - } - fun Collection.filter(batch: GroupBatch): GroupBatch? { if (isEmpty()) { return batch @@ -78,29 +61,6 @@ fun Collection.filter(batch: GroupBatch): GroupBatch? { return null } -fun EventId.toProto(): EventID = EventID.newBuilder().also { - it.id = id - it.bookName = book - it.scope = scope - it.startTimestamp = timestamp.toTimestamp() -}.build() - -fun MessageId.toProto(book: String, sessionGroup: String): MessageID = MessageID.newBuilder().also { - it.bookName = book - it.direction = if (direction == INCOMING) FIRST else SECOND - it.sequence = sequence - it.timestamp = timestamp.toTimestamp() - - it.addAllSubsequence(subsequence) - - it.connectionIdBuilder.also { connectionId -> - connectionId.sessionGroup = sessionGroup.ifBlank { sessionAlias } - connectionId.sessionAlias = sessionAlias - } -}.build() - -fun MessageId.toProto(groupBatch: GroupBatch): MessageID = toProto(groupBatch.book, groupBatch.sessionGroup) - private fun Collection?.verify(value: String): Boolean { if (isNullOrEmpty()) { return true @@ -126,4 +86,25 @@ private inline fun Collection?.verify( } return all { filter -> filter.checkFieldValue(firstGroup.messages.first().value()) } -} \ No newline at end of file +} + +fun GroupBatch.toByteArray(): ByteArray = Unpooled.buffer().run { + GroupBatchCodec.encode(this@toByteArray, this@run) + ByteArray(readableBytes()).apply(::readBytes) +} + +fun ByteBuf.toByteArray(): ByteArray = ByteArray(readableBytes()) + .apply(::readBytes).also { resetReaderIndex() } + +val Direction.proto: ProtoDirection + get() = when (this) { + INCOMING -> FIRST + OUTGOING -> SECOND + } + +val ProtoDirection.transport: Direction + get() = when (this) { + FIRST -> INCOMING + SECOND -> OUTGOING + else -> error("Unsupported $this direction in the th2 transport protocol") + } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/CollectionBuilder.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/CollectionBuilder.kt new file mode 100644 index 000000000..d55b47502 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/CollectionBuilder.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders + +class CollectionBuilder { + private val elements: MutableList = mutableListOf() + + val size: Int + get() = elements.size + + fun isEmpty(): Boolean = elements.isEmpty() + + fun add(el: T): CollectionBuilder = apply { + elements += el + } + + fun addAll(vararg els: T): CollectionBuilder = apply { + for (el in els) { + add(el) + } + } + + fun addAll(elements: Collection): CollectionBuilder = apply { + this.elements.addAll(elements) + } + + fun build(): List = elements +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/MapBuilder.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/MapBuilder.kt new file mode 100644 index 000000000..d85f6c43d --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/MapBuilder.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders + +class MapBuilder( + private val innerMap: MutableMap = hashMapOf(), +) { + fun put(key: K, value: V): MapBuilder = apply { + innerMap[key] = value + } + fun putAll(from: Map): MapBuilder = apply { + innerMap.putAll(from) + } + fun build(): Map { + return innerMap + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/value/ValueUtils.kt b/src/main/kotlin/com/exactpro/th2/common/value/ValueUtils.kt index ebc87831d..0bddbfef1 100644 --- a/src/main/kotlin/com/exactpro/th2/common/value/ValueUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/value/ValueUtils.kt @@ -15,6 +15,7 @@ */ @file:JvmName("ValueUtils") +@file:Suppress("unused") package com.exactpro.th2.common.value @@ -26,11 +27,12 @@ import com.exactpro.th2.common.grpc.NullValue.NULL_VALUE import com.exactpro.th2.common.grpc.Value import com.exactpro.th2.common.grpc.Value.KindCase.SIMPLE_VALUE import com.exactpro.th2.common.grpc.ValueOrBuilder +import com.exactpro.th2.common.message.addField import java.math.BigDecimal import java.math.BigInteger fun nullValue(): Value = Value.newBuilder().setNullValue(NULL_VALUE).build() -fun listValue() : ListValue.Builder = ListValue.newBuilder() +fun listValue(): ListValue.Builder = ListValue.newBuilder() fun Value.getString(): String? = takeIf { kindCase == SIMPLE_VALUE }?.simpleValue fun Value.getInt(): Int? = this.getString()?.toIntOrNull() @@ -39,34 +41,53 @@ fun Value.getDouble(): Double? = this.getString()?.toDoubleOrNull() fun Value.getBigInteger(): BigInteger? = this.getString()?.toBigIntegerOrNull() fun Value.getBigDecimal(): BigDecimal? = this.getString()?.toBigDecimalOrNull() fun Value.getMessage(): Message? = takeIf(Value::hasMessageValue)?.messageValue -fun Value.getList() : List? = takeIf(Value::hasListValue)?.listValue?.valuesList +fun Value.getList(): List? = takeIf(Value::hasListValue)?.listValue?.valuesList -fun Value.Builder.updateList(updateFunc: ListValue.Builder.() -> ListValueOrBuilder) : Value.Builder = apply { check(hasListValue()) { "Can not find list value" }; updateOrAddList(updateFunc) } -fun Value.Builder.updateString(updateFunc: String.() -> String) : ValueOrBuilder = apply { simpleValue = updateFunc(simpleValue ?: throw NullPointerException("Can not find simple value")) } -fun Value.Builder.updateMessage(updateFunc: Message.Builder.() -> MessageOrBuilder) : Value.Builder = apply { check(hasMessageValue()) { "Can not find message value" }; updateOrAddMessage(updateFunc) } +fun Value.Builder.updateList(updateFunc: ListValue.Builder.() -> ListValueOrBuilder): Value.Builder = + apply { check(hasListValue()) { "Can not find list value" }; updateOrAddList(updateFunc) } -fun Value.Builder.updateOrAddList(updateFunc: ListValue.Builder.() -> ListValueOrBuilder) : Value.Builder = apply { updateFunc(listValueBuilder).also { - when (it) { - is ListValue -> listValue = it - is ListValue.Builder -> setListValue(it) - else -> error("Can not set list value. Wrong type = ${it::class.java.canonicalName}") +fun Value.Builder.updateString(updateFunc: String.() -> String): ValueOrBuilder = + apply { simpleValue = updateFunc(simpleValue ?: throw NullPointerException("Can not find simple value")) } + +fun Value.Builder.updateMessage(updateFunc: Message.Builder.() -> MessageOrBuilder): Value.Builder = + apply { check(hasMessageValue()) { "Can not find message value" }; updateOrAddMessage(updateFunc) } + +fun Value.Builder.updateOrAddList(updateFunc: ListValue.Builder.() -> ListValueOrBuilder): Value.Builder = apply { + updateFunc(listValueBuilder).also { + when (it) { + is ListValue -> listValue = it + is ListValue.Builder -> setListValue(it) + else -> error("Can not set list value. Wrong type = ${it::class.java.canonicalName}") + } } -} } -fun Value.Builder.updateOrAddMessage(updateFunc: Message.Builder.() -> MessageOrBuilder) : Value.Builder = apply { updateFunc(messageValueBuilder).also { - when(it) { - is Message -> messageValue = it - is Message.Builder -> setMessageValue(it) - else -> error("Can not set message value. Wrong type = ${it::class.java.canonicalName}") +} + +fun Value.Builder.updateOrAddMessage(updateFunc: Message.Builder.() -> MessageOrBuilder): Value.Builder = apply { + updateFunc(messageValueBuilder).also { + when (it) { + is Message -> messageValue = it + is Message.Builder -> setMessageValue(it) + else -> error("Can not set message value. Wrong type = ${it::class.java.canonicalName}") + } } -} } -fun Value.Builder.updateOrAddString(updateFunc: String.() -> String) : Value.Builder = apply { updateFunc(simpleValue).also { simpleValue = it } } +} + +fun Value.Builder.updateOrAddString(updateFunc: String.() -> String): Value.Builder = + apply { updateFunc(simpleValue).also { simpleValue = it } } -fun ListValue.Builder.update(i: Int, updateFunc: Value.Builder.() -> ValueOrBuilder?): ListValue.Builder = apply { updateFunc(getValuesBuilder(i))?.let { setValues(i, it.toValue()) } } -fun ListValue.Builder.updateList(i: Int, updateFunc: ListValue.Builder.() -> ListValueOrBuilder) : ListValue.Builder = apply { getValuesBuilder(i).updateList(updateFunc)} -fun ListValue.Builder.updateString(i: Int, updateFunc: String.() -> String) : ListValue.Builder = apply { getValuesBuilder(i).updateString(updateFunc) } -fun ListValue.Builder.updateMessage(i: Int, updateFunc: Message.Builder.() -> MessageOrBuilder) : ListValue.Builder = apply { getValuesBuilder(i).updateMessage(updateFunc) } +fun ListValue.Builder.update(i: Int, updateFunc: Value.Builder.() -> ValueOrBuilder?): ListValue.Builder = + apply { updateFunc(getValuesBuilder(i))?.let { setValues(i, it.toValue()) } } -fun ListValue.Builder.updateOrAdd(i: Int, updateFunc: (Value.Builder?) -> ValueOrBuilder?) : ListValue.Builder = apply { +fun ListValue.Builder.updateList(i: Int, updateFunc: ListValue.Builder.() -> ListValueOrBuilder): ListValue.Builder = + apply { getValuesBuilder(i).updateList(updateFunc) } + +fun ListValue.Builder.updateString(i: Int, updateFunc: String.() -> String): ListValue.Builder = + apply { getValuesBuilder(i).updateString(updateFunc) } + +fun ListValue.Builder.updateMessage(i: Int, updateFunc: Message.Builder.() -> MessageOrBuilder): ListValue.Builder = + apply { getValuesBuilder(i).updateMessage(updateFunc) } + +fun ListValue.Builder.updateOrAdd(i: Int, updateFunc: (Value.Builder?) -> ValueOrBuilder?): ListValue.Builder = apply { updateFunc(if (i < valuesCount) getValuesBuilder(i) else null).also { while (i < valuesCount) { addValues(nullValue()) @@ -74,23 +95,36 @@ fun ListValue.Builder.updateOrAdd(i: Int, updateFunc: (Value.Builder?) -> ValueO add(i, it) } } -fun ListValue.Builder.updateOrAddList(i: Int, updateFunc: (ListValue.Builder?) -> ListValueOrBuilder) : ListValue.Builder = apply { updateOrAdd(i) { it?.updateOrAddList(updateFunc) ?: updateFunc(null)?.toValue() }} -fun ListValue.Builder.updateOrAddString(i: Int, updateFunc: (String?) -> String) : ListValue.Builder = apply { updateOrAdd(i) { it?.updateOrAddString(updateFunc) ?: updateFunc(null)?.toValue() } } -fun ListValue.Builder.updateOrAddMessage(i: Int, updateFunc: (Message.Builder?) -> MessageOrBuilder) : ListValue.Builder = apply { updateOrAdd(i) { it?.updateOrAddMessage(updateFunc) ?: updateFunc(null)?.toValue() }} -operator fun ListValueOrBuilder.get(i: Int) : Value = getValues(i) -operator fun ListValue.Builder.set(i: Int, value: Any?): ListValue.Builder = setValues(i, value?.toValue() ?: nullValue()) +fun ListValue.Builder.updateOrAddList( + i: Int, + updateFunc: (ListValue.Builder?) -> ListValueOrBuilder +): ListValue.Builder = apply { updateOrAdd(i) { it?.updateOrAddList(updateFunc) ?: updateFunc(null).toValue() } } + +fun ListValue.Builder.updateOrAddString(i: Int, updateFunc: (String?) -> String): ListValue.Builder = + apply { updateOrAdd(i) { it?.updateOrAddString(updateFunc) ?: updateFunc(null).toValue() } } -fun ListValue.Builder.add(value: Any?) : ListValue.Builder = apply { addValues(value?.toValue() ?: nullValue()) } -fun ListValue.Builder.add(i: Int, value: Any?) : ListValue.Builder = apply { addValues(i, value?.toValue() ?: nullValue()) } +fun ListValue.Builder.updateOrAddMessage( + i: Int, + updateFunc: (Message.Builder?) -> MessageOrBuilder +): ListValue.Builder = apply { updateOrAdd(i) { it?.updateOrAddMessage(updateFunc) ?: updateFunc(null).toValue() } } -fun Any.toValue(): Value = when (this) { +operator fun ListValueOrBuilder.get(i: Int): Value = getValues(i) +operator fun ListValue.Builder.set(i: Int, value: Any?): ListValue.Builder = + setValues(i, value?.toValue() ?: nullValue()) + +fun ListValue.Builder.add(value: Any?): ListValue.Builder = apply { addValues(value?.toValue() ?: nullValue()) } +fun ListValue.Builder.add(i: Int, value: Any?): ListValue.Builder = + apply { addValues(i, value?.toValue() ?: nullValue()) } + +fun Any?.toValue(): Value = when (this) { is Message -> toValue() is Message.Builder -> toValue() is ListValue -> toValue() is ListValue.Builder -> toValue() is Iterator<*> -> toValue() is Iterable<*> -> toValue() + is Map<*, *> -> toValue() is Array<*> -> toValue() is BooleanArray -> toValue() is ByteArray -> toValue() @@ -102,6 +136,7 @@ fun Any.toValue(): Value = when (this) { is DoubleArray -> toValue() is Value -> this is Value.Builder -> toValue() + null -> nullValue() else -> toString().toValue() } @@ -110,13 +145,19 @@ fun String.toValue(): Value = Value.newBuilder().setSimpleValue(this).build() fun Message.toValue(): Value = Value.newBuilder().setMessageValue(this).build() fun Message.Builder.toValue(): Value = Value.newBuilder().setMessageValue(this).build() -fun ListValue.toValue() : Value = Value.newBuilder().setListValue(this).build() -fun ListValue.Builder.toValue() : Value = Value.newBuilder().setListValue(this).build() +fun ListValue.toValue(): Value = Value.newBuilder().setListValue(this).build() +fun ListValue.Builder.toValue(): Value = Value.newBuilder().setListValue(this).build() -fun Value.Builder.toValue() : Value = this.build() +fun Value.Builder.toValue(): Value = this.build() fun Iterator<*>.toValue(): Value = toListValue().toValue() fun Iterable<*>.toValue(): Value = iterator().toValue() +fun Map<*, *>.toValue(): Value = Message.newBuilder().apply { + forEach { (key, value) -> + addField(key.toString(), value.toValue()) + } +}.toValue() + fun Array<*>.toValue(): Value = iterator().toValue() fun BooleanArray.toValue(): Value = toTypedArray().toValue() @@ -129,14 +170,14 @@ fun FloatArray.toValue(): Value = toTypedArray().toValue() fun DoubleArray.toValue(): Value = toTypedArray().toValue() -fun Iterator<*>.toListValue() : ListValue = listValue().also { list -> +fun Iterator<*>.toListValue(): ListValue = listValue().also { list -> forEach { - it?.toValue().run(list::addValues) + it.toValue().run(list::addValues) } }.build() -fun Iterable<*>.toListValue() : ListValue = iterator().toListValue() -fun Array<*>.toListValue() : ListValue = iterator().toListValue() +fun Iterable<*>.toListValue(): ListValue = iterator().toListValue() +fun Array<*>.toListValue(): ListValue = iterator().toListValue() fun BooleanArray.toListValue(): ListValue = toTypedArray().toListValue() fun ByteArray.toListValue(): ListValue = toTypedArray().toListValue() diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt deleted file mode 100644 index b6498baf5..000000000 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CleanableTest.kt +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2023 Exactpro (Exactpro Systems Limited) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport - -import io.netty.buffer.Unpooled -import org.junit.jupiter.api.Test -import java.time.Instant -import kotlin.test.assertEquals -import kotlin.test.assertNotEquals - -class CleanableTest { - - private val filledMessageId = MessageId( - "sessionAlias", - Direction.OUTGOING, - 1, - mutableListOf(2, 3), - Instant.now(), - ) - private val filledEventId = EventId( - "id", - "book", - "scope", - Instant.now(), - ) - - @Test - fun `event id clean test`() { - val empty = EventId.DEFAULT_INSTANCE - - val mutable = EventId.newMutable().apply { - id = "id" - book = "book" - scope = "scope" - timestamp = Instant.now() - } - assertNotEquals(empty, mutable) - - mutable.clean() - assertEquals(empty, mutable) - } - - @Test - fun `message id clean test`() { - val empty = MessageId.DEFAULT_INSTANCE - - val mutable = MessageId.newMutable().apply { - sessionAlias = "sessionAlias" - direction = Direction.OUTGOING - sequence = 1 - subsequence = mutableListOf(2, 3) - timestamp = Instant.now() - } - assertNotEquals(empty, mutable) - - mutable.clean() - assertEquals(empty, mutable) - } - - @Test - fun `raw message clean test`() { - val empty = RawMessage() - - val mutable = RawMessage.newMutable().apply { - id = filledMessageId - eventId = filledEventId - metadata["property"] = "value" - protocol = "protocol" - body.writeByte(64) - } - assertNotEquals(empty, mutable) - - mutable.clean() - assertEquals(empty, mutable) - - val softMutable = RawMessage.newSoftMutable().apply { - id = filledMessageId - eventId = filledEventId - metadata["property"] = "value" - protocol = "protocol" - body = Unpooled.buffer().writeByte(64) - } - assertNotEquals(empty, softMutable) - - softMutable.softClean() - assertEquals(empty, softMutable) - } - - @Test - fun `parsed message clean test`() { - val empty = ParsedMessage() - - val mutable = ParsedMessage.newMutable().apply { - id = filledMessageId - eventId = filledEventId - metadata["property"] = "value" - protocol = "protocol" - type = "type" - } - assertNotEquals(empty, mutable) - - mutable.clean() - assertEquals(empty, mutable) - - val softMutable = ParsedMessage.newSoftMutable().apply { - id = filledMessageId - eventId = filledEventId - metadata["property"] = "value" - protocol = "protocol" - type = "type" - } - assertNotEquals(empty, softMutable) - - softMutable.softClean() - assertEquals(empty, softMutable) - } - - @Test - fun `message group clean test`() { - val empty = MessageGroup() - - val mutable = MessageGroup.newMutable().apply { - messages.add(RawMessage()) - } - assertNotEquals(empty, mutable) - - mutable.clean() - assertEquals(empty, mutable) - - mutable.messages.add(ParsedMessage()) - assertNotEquals(empty, mutable) - - mutable.softClean() - assertEquals(empty, mutable) - } - - @Test - fun `group batch clean test`() { - val empty = GroupBatch() - - val mutable = GroupBatch.newMutable().apply { - groups.add(MessageGroup()) - } - assertNotEquals(empty, mutable) - - mutable.clean() - assertEquals(empty, mutable) - - mutable.groups.add(MessageGroup()) - assertNotEquals(empty, mutable) - - mutable.softClean() - assertEquals(empty, mutable) - } -} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt index 5ce8f6061..f76694250 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt @@ -16,6 +16,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport +import io.netty.buffer.ByteBufUtil import io.netty.buffer.Unpooled import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -73,11 +74,7 @@ class CodecsTest { ), protocol = "proto3", type = "some-type", - body = mutableMapOf( - "simple" to 1, - "list" to listOf(1, 2, 3), - "map" to mapOf("abc" to "cde") - ) + rawBody = Unpooled.buffer().apply { writeCharSequence("{}", Charsets.UTF_8) } ) val batch = GroupBatch( @@ -91,4 +88,37 @@ class CodecsTest { assertEquals(batch, decodedBatch) } + + @Test + fun `raw body is updated in parsed message when body is changed`() { + val parsedMessage = ParsedMessage.builder().apply { + idBuilder() + .setSessionAlias("alias1") + .setDirection(Direction.INCOMING) + .setSequence(1) + .addSubsequence(1) + .setTimestamp(Instant.now()) + setType("test") + setBody( + linkedMapOf( + "field" to 42, + "another" to "test_data", + ) + ) + }.build() + + val dest = Unpooled.buffer() + ParsedMessageCodec.encode(parsedMessage, dest) + val decoded = ParsedMessageCodec.decode(dest) + assertEquals(0, dest.readableBytes()) { "unexpected bytes left: ${ByteBufUtil.hexDump(dest)}" } + + assertEquals(parsedMessage, decoded, "unexpected parsed result decoded") + assertEquals( + Unpooled.buffer().apply { + writeCharSequence("{\"field\":42,\"another\":\"test_data\"}", Charsets.UTF_8) + }, + decoded.rawBody, + "unexpected raw body", + ) + } } \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterIntegrationTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterIntegrationTest.kt index e27d3c9a3..ffbbd5702 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterIntegrationTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterIntegrationTest.kt @@ -1,5 +1,6 @@ /* * Copyright 2023 Exactpro (Exactpro Systems Limited) + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -49,7 +50,10 @@ class TransportGroupBatchRouterIntegrationTest { val counter = CountDownLatch(1) val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } try { - secondRouter.sendExclusive(monitor.queue, GroupBatch()) + secondRouter.sendExclusive(monitor.queue, GroupBatch.builder() + .setBook("") + .setSessionGroup("") + .build()) assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } } finally { @@ -80,7 +84,10 @@ class TransportGroupBatchRouterIntegrationTest { val monitor = firstRouter.subscribeExclusive { _, _ -> counter.countDown() } try { - secondRouter.sendExclusive(monitor.queue, GroupBatch()) + secondRouter.sendExclusive(monitor.queue, GroupBatch.builder() + .setBook("") + .setSessionGroup("") + .build()) assertTrue("Message is not received") { counter.await(1, TimeUnit.SECONDS) } } finally { diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt index 1787f19a4..25fcd7976 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt @@ -32,6 +32,7 @@ import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.function.Executable import org.mockito.kotlin.* +import java.time.Instant class TransportGroupBatchRouterTest { private val connectionConfiguration = ConnectionManagerConfiguration() @@ -288,7 +289,12 @@ class TransportGroupBatchRouterTest { MessageGroup( mutableListOf( ParsedMessage( - MessageId(SESSION_ALIAS), + MessageId.builder() + .setSessionAlias(SESSION_ALIAS) + .setDirection(Direction.INCOMING) + .setSequence(1) + .setTimestamp(Instant.now()) + .build(), type = messageType ) ) @@ -316,7 +322,12 @@ class TransportGroupBatchRouterTest { MessageGroup( mutableListOf( RawMessage( - MessageId(SESSION_ALIAS), + MessageId.builder() + .setSessionAlias(SESSION_ALIAS) + .setDirection(Direction.INCOMING) + .setSequence(1) + .setTimestamp(Instant.now()) + .build(), body = Unpooled.wrappedBuffer(byteArrayOf(1, 2, 3)) ) ) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt index 4fb81b696..40b4c6819 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt @@ -16,15 +16,25 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.* +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.BOOK_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.DIRECTION_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration -import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation.* +import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation.EQUAL +import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation.NOT_EMPTY +import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation.NOT_EQUAL +import com.exactpro.th2.common.schema.message.configuration.FieldFilterOperation.WILDCARD import com.exactpro.th2.common.schema.message.configuration.MqRouterFilterConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.util.emptyMultiMap import org.apache.commons.collections4.MultiMapUtils -import org.junit.jupiter.api.Test -import kotlin.test.assertEquals +import org.junit.jupiter.api.DynamicTest +import org.junit.jupiter.api.TestFactory +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import java.time.Instant import kotlin.test.assertNull import kotlin.test.assertSame @@ -44,14 +54,16 @@ class TransportUtilsTest { private val routerFilters = listOf( MqRouterFilterConfiguration( MultiMapUtils.newListValuedHashMap().apply { - putAll(BOOK_KEY, listOf( - FieldFilterConfiguration(BOOK_KEY, bookA, EQUAL), - FieldFilterConfiguration(BOOK_KEY, "*A", WILDCARD), - FieldFilterConfiguration(BOOK_KEY, null, NOT_EMPTY), - FieldFilterConfiguration(BOOK_KEY, bookB, NOT_EQUAL) - )) + putAll( + BOOK_KEY, listOf( + FieldFilterConfiguration(BOOK_KEY, bookA, EQUAL), + FieldFilterConfiguration(BOOK_KEY, "*A", WILDCARD), + FieldFilterConfiguration(BOOK_KEY, null, NOT_EMPTY), + FieldFilterConfiguration(BOOK_KEY, bookB, NOT_EQUAL) + ) + ) put(SESSION_GROUP_KEY, FieldFilterConfiguration(SESSION_GROUP_KEY, "*A", WILDCARD)) - put(SESSION_ALIAS_KEY, FieldFilterConfiguration(SESSION_ALIAS_KEY, null, EMPTY)) + put(SESSION_ALIAS_KEY, FieldFilterConfiguration(SESSION_ALIAS_KEY, null, NOT_EMPTY)) put(MESSAGE_TYPE_KEY, FieldFilterConfiguration(MESSAGE_TYPE_KEY, null, NOT_EMPTY)) put(DIRECTION_KEY, FieldFilterConfiguration(DIRECTION_KEY, directionB, NOT_EQUAL)) }, @@ -59,14 +71,16 @@ class TransportUtilsTest { ), MqRouterFilterConfiguration( MultiMapUtils.newListValuedHashMap().apply { - putAll(BOOK_KEY, listOf( - FieldFilterConfiguration(BOOK_KEY, bookB, EQUAL), - FieldFilterConfiguration(BOOK_KEY, "*B", WILDCARD), - FieldFilterConfiguration(BOOK_KEY, null, NOT_EMPTY), - FieldFilterConfiguration(BOOK_KEY, bookA, NOT_EQUAL) - )) + putAll( + BOOK_KEY, listOf( + FieldFilterConfiguration(BOOK_KEY, bookB, EQUAL), + FieldFilterConfiguration(BOOK_KEY, "*B", WILDCARD), + FieldFilterConfiguration(BOOK_KEY, null, NOT_EMPTY), + FieldFilterConfiguration(BOOK_KEY, bookA, NOT_EQUAL) + ) + ) put(SESSION_GROUP_KEY, FieldFilterConfiguration(SESSION_GROUP_KEY, "*B", WILDCARD)) - put(SESSION_ALIAS_KEY, FieldFilterConfiguration(SESSION_ALIAS_KEY, null, EMPTY)) + put(SESSION_ALIAS_KEY, FieldFilterConfiguration(SESSION_ALIAS_KEY, null, NOT_EMPTY)) put(MESSAGE_TYPE_KEY, FieldFilterConfiguration(MESSAGE_TYPE_KEY, null, NOT_EMPTY)) put(DIRECTION_KEY, FieldFilterConfiguration(DIRECTION_KEY, directionA, NOT_EQUAL)) }, @@ -74,43 +88,109 @@ class TransportUtilsTest { ) ) - @Test - fun `empty filter test`() { - val group = GroupBatch() - assertSame(group, listOf().filter(group)) - - group.book = bookB - group.sessionGroup = groupA + @ParameterizedTest + @ValueSource(strings = ["", "data"]) + fun `empty filter test`(strValue: String) { + val group = GroupBatch.builder() + .setBook(strValue) + .setSessionGroup(strValue) + .build() assertSame(group, listOf().filter(group)) } - @Test - fun `filter test`() { - val batch = GroupBatch.newMutable() - assertNull(routerFilters.filter(batch)) - - batch.book = bookA - assertNull(routerFilters.filter(batch)) - batch.sessionGroup = groupA - assertNull(routerFilters.filter(batch)) - val group = MessageGroup.newMutable() - batch.groups.add(group) - assertNull(routerFilters.filter(batch)) - - val parsedMessage = ParsedMessage.newMutable() - group.messages.add(parsedMessage) - assertNull(routerFilters.filter(batch)) - - parsedMessage.type = msgType - assertNull(routerFilters.filter(batch)) - - parsedMessage.id = MessageId(direction = Direction.OUTGOING) - assertEquals(batch, routerFilters.filter(batch)) - - parsedMessage.id = MessageId(direction = Direction.INCOMING) - batch.sessionGroup = groupB - batch.book = bookB - assertEquals(batch, routerFilters.filter(batch)) + @TestFactory + fun `filter test`(): Collection { + return listOf( + DynamicTest.dynamicTest("empty batch") { + val batch = GroupBatch.builder() + .setBook("") + .setSessionGroup("") + .build() + assertNull(routerFilters.filter(batch)) + }, + DynamicTest.dynamicTest("only book match") { + val batch = GroupBatch.builder() + .setBook(bookA) + .setSessionGroup("") + .build() + assertNull(routerFilters.filter(batch)) + }, + DynamicTest.dynamicTest("only book and group match") { + val batch = GroupBatch.builder() + .setBook(bookA) + .setSessionGroup(groupA) + .build() + assertNull(routerFilters.filter(batch)) + }, + DynamicTest.dynamicTest("with partial message match") { + val batch = GroupBatch.builder() + .setBook(bookA) + .setSessionGroup(groupA) + .addGroup( + MessageGroup.builder() + .addMessage( + ParsedMessage.builder() + .setId( + MessageId.builder() + .setSessionAlias("") + .setDirection(Direction.OUTGOING) + .setSequence(1) + .setTimestamp(Instant.now()) + .build() + ) + .setType(msgType) + .setBody(emptyMap()) + .build() + ) + .build() + ).build() + assertNull(routerFilters.filter(batch)) + }, + DynamicTest.dynamicTest("full match for A") { + val batch = GroupBatch.builder() + .setBook(bookA) + .setSessionGroup(groupA) + .addGroup( + MessageGroup.builder() + .addMessage( + ParsedMessage.builder() + .setType(msgType) + .setBody(emptyMap()) + .apply { + idBuilder() + .setSessionAlias("alias") + .setDirection(Direction.OUTGOING) + .setSequence(1) + .setTimestamp(Instant.now()) + }.build() + ) + .build() + ).build() + assertSame(batch, routerFilters.filter(batch)) + }, + DynamicTest.dynamicTest("full match for B") { + val batch = GroupBatch.builder() + .setBook(bookB) + .setSessionGroup(groupB) + .addGroup( + MessageGroup.builder() + .addMessage( + ParsedMessage.builder() + .setType(msgType) + .setBody(emptyMap()) + .apply { + idBuilder() + .setSessionAlias("alias") + .setDirection(Direction.INCOMING) + .setSequence(1) + .setTimestamp(Instant.now()) + }.build() + ) + .build() + ).build() + assertSame(batch, routerFilters.filter(batch)) + }, + ) } } \ No newline at end of file From e96e5cc3c13fe98a319ff0307c731fd127eaee54 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 19 Jun 2023 10:36:17 +0400 Subject: [PATCH 124/154] Added suppressions.xml --- build.gradle | 9 ++++++--- suppressions.xml | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 suppressions.xml diff --git a/build.gradle b/build.gradle index 8b5786810..7086d1fb1 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.1.0-separate-executor-4783246997-dda4d35-SNAPSHOT' + cradleVersion = '5.1.0-separate-executor-5308573193-b16f7f7-SNAPSHOT' junitVersion = '5.8.2' sharedDir = file("${project.rootDir}/shared") } @@ -197,7 +197,10 @@ dependencies { def autoValueVersion = '1.10.1' implementation "com.google.auto.value:auto-value-annotations:$autoValueVersion" - kapt "com.google.auto.value:auto-value:$autoValueVersion" + kapt("com.google.auto.value:auto-value:$autoValueVersion") { + //FIXME: Updated library because it is fat jar + // auto-value-1.10.1.jar/META-INF/maven/com.google.guava/guava/pom.xml (pkg:maven/com.google.guava/guava@31.1-jre, cpe:2.3:a:google:guava:31.1:*:*:*:*:*:*:*) : CVE-2023-2976, CVE-2020-8908 + } //this is required to add generated bridge classes for kotlin default constructors implementation(files("$buildDir/tmp/kapt3/classes/main")) @@ -287,7 +290,7 @@ clean { dependencyCheck { formats = ['SARIF', 'JSON', 'HTML'] failBuildOnCVSS = 5 - + suppressionFile = file('suppressions.xml') analyzers { assemblyEnabled = false nugetconfEnabled = false diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 000000000..50f2320cb --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,16 @@ + + + + + + ^pkg:maven/com\.exactpro\.th2/task-utils@.*$ + cpe:/a:utils_project:utils + + + + + .*/auto-value-1.10.1.jar/META-INF/maven/com.google.guava/guava/pom.xml + CVE-2023-2976 + CVE-2020-8908 + + \ No newline at end of file From 63bc4eadccf51e10d7965adab4c68d4742b8b47c Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Thu, 22 Jun 2023 19:36:17 +0400 Subject: [PATCH 125/154] [TH2-4887] updated dependence --- build.gradle | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 29a58a588..a1a03c920 100644 --- a/build.gradle +++ b/build.gradle @@ -30,8 +30,8 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.1.0-separate-executor-5308573193-b16f7f7-SNAPSHOT' - junitVersion = '5.8.2' + cradleVersion = '5.1.0-separate-executor-5334417446-8a1a37d-SNAPSHOT' + junitVersion = '5.9.3' } repositories { @@ -167,8 +167,8 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:4.2.0") - api('com.exactpro.th2:grpc-common:4.2.0-dev') { + api platform("com.exactpro.th2:bom:4.4.0-update-libs-5335052162-8affa4a-SNAPSHOT") + api('com.exactpro.th2:grpc-common:4.3.0-TH2-4887-vulnerabilities-5346500826-f180a19-SNAPSHOT') { because('protobuf transport is main now, this dependnecy should be moved to grpc, mq protobuf modules after splitting') } api("com.exactpro.th2:cradle-core:${cradleVersion}") { @@ -182,7 +182,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-generator-annprocess:0.9' implementation 'com.google.protobuf:protobuf-java-util' - implementation 'com.exactpro.th2:grpc-service-generator:3.2.2' + implementation 'com.exactpro.th2:grpc-service-generator:3.4.0-TH2-4887-vulnerabilities-5346040251-1bed6cd-SNAPSHOT' implementation "com.exactpro.th2:cradle-cassandra:${cradleVersion}" def autoValueVersion = '1.10.1' From 4a0315ce037c67644f3c2236fc5f50fbe486afa1 Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Fri, 21 Jul 2023 12:30:28 +0000 Subject: [PATCH 126/154] [TH2-5002] Book filtering for 5 (#273) * Book, session group and protocol message filtering added. * Pin filters behaviour changed: conditions inside the message and meta data now combined as "and" Tests added Co-authored-by: Oleg Smelov --- README.md | 7 +- gradle.properties | 2 +- .../strategy/impl/AbstractFilterStrategy.java | 4 +- .../impl/AbstractTh2MsgFilterStrategy.java | 17 +- .../strategy/impl/AnyMessageFilterStrategy.kt | 10 +- .../impl/TestAnyMessageFilterStrategy.kt | 478 +++++++++++++++++- 6 files changed, 480 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index eb9f7f38b..ee1034984 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (5.2.0) +# th2 common library (Java) (5.2.2) ## Usage @@ -361,6 +361,11 @@ dependencies { ## Release notes +### 5.2.2 + +#### Changed: ++ Book, session group and protocol message filtering added. + ### 5.2.1 #### Changed: diff --git a/gradle.properties b/gradle.properties index b8fc06ddf..147694234 100644 --- a/gradle.properties +++ b/gradle.properties @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.2.1 +release_version=5.2.2 description = 'th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j diff --git a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java index fee14236e..7523d6208 100644 --- a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java +++ b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -52,7 +52,7 @@ public boolean verify(T message, List routerFilters) { protected abstract Map getFields(T message); private boolean checkValues(Map messageFields, MultiValuedMap fieldFilters) { - return fieldFilters.isEmpty() || fieldFilters.keys().stream().anyMatch(fieldName -> { + return fieldFilters.isEmpty() || fieldFilters.keys().stream().allMatch(fieldName -> { String messageValue = messageFields.get(fieldName); Collection filters = fieldFilters.get(fieldName); return !filters.isEmpty() && filters.stream().allMatch(filter -> FieldValueChecker.checkFieldValue(filter, messageValue)); diff --git a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java index 2d0fe5964..0df099e30 100644 --- a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java +++ b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -15,7 +15,6 @@ package com.exactpro.th2.common.schema.filter.strategy.impl; - import com.exactpro.th2.common.grpc.Message; import com.exactpro.th2.common.grpc.MessageID; import com.exactpro.th2.common.grpc.MessageMetadata; @@ -23,12 +22,14 @@ import java.util.Map; import java.util.stream.Collectors; - public abstract class AbstractTh2MsgFilterStrategy extends AbstractFilterStrategy { + public static final String BOOK_KEY = "book"; + public static final String SESSION_GROUP_KEY = "session_group"; public static final String SESSION_ALIAS_KEY = "session_alias"; public static final String MESSAGE_TYPE_KEY = "message_type"; public static final String DIRECTION_KEY = "direction"; + public static final String PROTOCOL_KEY = "protocol"; @Override public Map getFields(com.google.protobuf.Message message) { @@ -41,10 +42,15 @@ public Map getFields(com.google.protobuf.Message message) { .map(entry -> Map.entry(entry.getKey(), entry.getValue().getSimpleValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + String sessionGroup = messageID.getConnectionId().getSessionGroup(); + String sessionAlias = messageID.getConnectionId().getSessionAlias(); var metadataMsgFields = Map.of( + BOOK_KEY, messageID.getBookName(), + SESSION_GROUP_KEY, sessionGroup.isEmpty() ? sessionAlias : sessionGroup, SESSION_ALIAS_KEY, messageID.getConnectionId().getSessionAlias(), MESSAGE_TYPE_KEY, metadata.getMessageType(), - DIRECTION_KEY, messageID.getDirection().name() + DIRECTION_KEY, messageID.getDirection().name(), + PROTOCOL_KEY, metadata.getProtocol() ); messageFields.putAll(metadataMsgFields); @@ -53,5 +59,4 @@ public Map getFields(com.google.protobuf.Message message) { } public abstract Message parseMessage(com.google.protobuf.Message message); - -} +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt index d39f229d1..090399abd 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,14 +32,22 @@ class AnyMessageFilterStrategy : AbstractFilterStrategy() { result.putAll(message.message.fieldsMap.mapValues { it.value.simpleValue }) val metadata = message.message.metadata + result.putAll(metadata.propertiesMap) + result[AbstractTh2MsgFilterStrategy.BOOK_KEY] = metadata.id.bookName + result[AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY] = metadata.id.connectionId.sessionGroup result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = metadata.id.connectionId.sessionAlias result[AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY] = metadata.messageType result[AbstractTh2MsgFilterStrategy.DIRECTION_KEY] = metadata.id.direction.name + result[AbstractTh2MsgFilterStrategy.PROTOCOL_KEY] = metadata.protocol } message.hasRawMessage() -> { val metadata = message.rawMessage.metadata + result.putAll(metadata.propertiesMap) + result[AbstractTh2MsgFilterStrategy.BOOK_KEY] = metadata.id.bookName + result[AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY] = metadata.id.connectionId.sessionGroup result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = metadata.id.connectionId.sessionAlias result[AbstractTh2MsgFilterStrategy.DIRECTION_KEY] = metadata.id.direction.name + result[AbstractTh2MsgFilterStrategy.PROTOCOL_KEY] = metadata.protocol } else -> throw IllegalStateException("Message has not messages: ${message.toJson()}") } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt index c1e7ada7e..c9c762bf2 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import com.exactpro.th2.common.event.bean.BaseTest.BOOK_NAME import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.Direction import com.exactpro.th2.common.grpc.RawMessage +import com.exactpro.th2.common.message.addField import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.toJson import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration @@ -35,6 +36,157 @@ import org.junit.jupiter.params.provider.MethodSource class TestAnyMessageFilterStrategy { private val strategy = AnyMessageFilterStrategy() + @ParameterizedTest + @MethodSource("multipleFiltersMatch") + fun `matches any filter`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + anyMessage, + listOf( + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("message_type", FieldFilterConfiguration( + fieldName = "message_type", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test" + )) + } + ), + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("direction", FieldFilterConfiguration( + fieldName = "direction", + operation = FieldFilterOperation.EQUAL, + expectedValue = "FIRST" + )) + } + ) + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("rawMessagesBothFilters") + fun `matches with multiple metadata filters`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + anyMessage, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("session_alias", FieldFilterConfiguration( + fieldName = "session_alias", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-alias" + )) + put("direction", FieldFilterConfiguration( + fieldName = "direction", + operation = FieldFilterOperation.EQUAL, + expectedValue = "FIRST" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("parsedMessagesBothFilters") + fun `matches with multiple message filters`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + anyMessage, + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put("test-field1", FieldFilterConfiguration( + fieldName = "test-field1", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value1" + )) + put("test-field2", FieldFilterConfiguration( + fieldName = "test-field2", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithProperties") + fun `matches with multiple properties filters`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + anyMessage, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("prop-field1", FieldFilterConfiguration( + fieldName = "prop-field1", + operation = FieldFilterOperation.EQUAL, + expectedValue = "prop-value1" + )) + put("prop-field2", FieldFilterConfiguration( + fieldName = "prop-field2", + operation = FieldFilterOperation.EQUAL, + expectedValue = "prop-value2" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithMessageAndMetadataFilters") + fun `matches with multiple message and metadata filters`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + anyMessage, + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put("test-field1", FieldFilterConfiguration( + fieldName = "test-field1", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value1" + )) + put("test-field2", FieldFilterConfiguration( + fieldName = "test-field2", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + )) + }, + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("message_type", FieldFilterConfiguration( + fieldName = "message_type", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test" + )) + put("direction", FieldFilterConfiguration( + fieldName = "direction", + operation = FieldFilterOperation.EQUAL, + expectedValue = "FIRST" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messageWithProtocol") + fun `matches protocol metadata filter`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + anyMessage, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("protocol", FieldFilterConfiguration( + fieldName = "protocol", + operation = FieldFilterOperation.EQUAL, + expectedValue = "HTTP" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + @ParameterizedTest @MethodSource("parsedMessages") fun `matches the parsed message by message type with single filter`(anyMessage: AnyMessage, expectMatch: Boolean) { @@ -84,48 +236,320 @@ class TestAnyMessageFilterStrategy { expectedValue = "test-alias" )) } - )) + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithSameFilterFields") + fun `miss matches with the same filter fields`(message: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + message, + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value1" + ) + ) + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + ) + ) + } + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithMultipleFiltersWithSameFilterField") + fun `matches with multiple filters with the same filter fields`(message: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + message, + listOf( + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value1" + ) + ) + } + ), + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + ) + ) + } + ) + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithMultipleSameFields") + fun `matches message with multiple fields with same name`(message: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + message, + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.NOT_EQUAL, + expectedValue = "test-value1" + ) + ) + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + ) + ) + } + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithOneProperty") + fun `matches message with properties`(message: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + message, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-property", FieldFilterConfiguration( + fieldName = "test-property", + operation = FieldFilterOperation.EQUAL, + expectedValue = "property-value" + ) + ) + } + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithPropertiesAndMetadata") + fun `matches message with properties and metadata`(message: AnyMessage, expectMatch: Boolean) { + val match = strategy.verify( + message, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-property", FieldFilterConfiguration( + fieldName = "test-property", + operation = FieldFilterOperation.EQUAL, + expectedValue = "property-value" + ) + ) + + put("message_type", FieldFilterConfiguration( + fieldName = "message_type", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test" + )) + } + ) + ) assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } } companion object { - private val PARSED_MESSAGE_MATCH = AnyMessage.newBuilder().setMessage( - message(BOOK_NAME, "test", Direction.FIRST, "test-alias") - ).build() + private fun simpleMessageBuilder(messageType: String, direction: Direction, sessionAlias: String): AnyMessage { + return AnyMessage.newBuilder().setMessage( + message(BOOK_NAME, messageType, direction, sessionAlias) + ).build() + } + + private fun simpleRawMessageBuilder(sessionAlias: String, directionValue: Direction): AnyMessage { + return AnyMessage.newBuilder().setRawMessage( + RawMessage.newBuilder().apply { + metadataBuilder.idBuilder.apply { + connectionIdBuilder.sessionAlias = sessionAlias + direction = directionValue + } + } + ).build() + } - private val RAW_MESSAGE_MATCH = AnyMessage.newBuilder().setRawMessage( - RawMessage.newBuilder().apply { - metadataBuilder.idBuilder.apply { - connectionIdBuilder.sessionAlias = "test-alias" - direction = Direction.FIRST + private fun messageWithFieldsBuilder(messageType: String, direction: Direction, fields: List>): AnyMessage { + return AnyMessage.newBuilder().setMessage( + message(BOOK_NAME, messageType, direction, "test-alias").apply { + fields.forEach { addField(it.first, it.second) } } - } - ).build() + ).build() + } - private val PARSED_MESSAGE_MISS_MATCH = AnyMessage.newBuilder().setMessage( - message(BOOK_NAME, "test1", Direction.SECOND, "test-alias1") - ).build() + private fun messageWithPropertiesBuilder(messageType: String, direction: Direction, properties: List>): AnyMessage { + return AnyMessage.newBuilder().setMessage( + message(BOOK_NAME, messageType, direction, "test-alias").apply { + properties.forEach { metadataBuilder.putProperties(it.first, it.second) } + } + ).build() + } + + private fun rawMessageWithOnePropertyBuilder(propertyKey: String, propertyValue: String): AnyMessage { + return AnyMessage.newBuilder().setRawMessage( + RawMessage.newBuilder().apply { + metadataBuilder.putProperties(propertyKey, propertyValue) + } + ).build() + } - private val RAW_MESSAGE_MISS_MATCH = AnyMessage.newBuilder().setRawMessage( - RawMessage.newBuilder().apply { - metadataBuilder.idBuilder.apply { - connectionIdBuilder.sessionAlias = "test-alias1" - direction = Direction.SECOND + private fun messageWithOnePropertyBuilder(messageType: String, propertyKey: String, propertyValue: String): AnyMessage { + return messageWithPropertiesBuilder(messageType, Direction.FIRST, listOf(Pair(propertyKey, propertyValue))) + } + + private fun messageWithProtocolBuilder(protocol: String): AnyMessage { + return AnyMessage.newBuilder().setMessage( + message(BOOK_NAME, "test", Direction.FIRST, "test-alias").apply { + metadataBuilder.protocol = protocol } - } - ).build() + ).build() + } @JvmStatic - fun parsedMessages(): List = listOf( - arguments(PARSED_MESSAGE_MATCH, true), - arguments(PARSED_MESSAGE_MISS_MATCH, false) + fun messageWithProtocol(): List = listOf( + arguments(messageWithProtocolBuilder("HTTP"), true), + arguments(messageWithProtocolBuilder("FTP"), false), ) @JvmStatic fun messages(): List = listOf( - arguments(RAW_MESSAGE_MATCH, true), - arguments(RAW_MESSAGE_MISS_MATCH, false) + arguments(simpleRawMessageBuilder("test-alias", Direction.FIRST), true), + arguments(simpleRawMessageBuilder("test-alias1", Direction.SECOND), false) ) + parsedMessages() + + @JvmStatic + fun parsedMessages(): List = listOf( + arguments(simpleMessageBuilder("test", Direction.FIRST, "test-alias"), true), + arguments(simpleMessageBuilder("test1", Direction.SECOND, "test-alias1"), false) + ) + + @JvmStatic + fun messagesWithProperties(): List = listOf( + arguments(messageWithPropertiesBuilder("test", Direction.FIRST, listOf( + Pair("prop-field1", "prop-value1"), Pair("prop-field2", "prop-value2"))), true), + arguments(messageWithPropertiesBuilder("test", Direction.FIRST, listOf( + Pair("prop-field1", "prop-value-wrong"), Pair("prop-field2", "prop-value2"))), false), + arguments(messageWithPropertiesBuilder("test", Direction.FIRST, listOf( + Pair("prop-field1", "prop-value1"), Pair("prop-field2", "prop-value-wrong"))), false), + arguments(messageWithPropertiesBuilder("test", Direction.FIRST, listOf( + Pair("prop-field1", "prop-value-wrong"), Pair("prop-field2", "prop-value-wrong"))), false) + ) + + @JvmStatic + fun messagesWithMultipleSameFields(): List = listOf( + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field", "test-value1"), Pair("test-field", "test-value2"))), true) + ) + + @JvmStatic + fun messagesWithSameFilterFields(): List = listOf( + arguments(messageWithFieldsBuilder("test", Direction.FIRST, listOf(Pair("test-field", "test-value1"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, listOf(Pair("test-field", "test-value2"))), false) + ) + + @JvmStatic + fun messagesWithMultipleFiltersWithSameFilterField(): List = listOf( + arguments(messageWithFieldsBuilder("test", Direction.FIRST, listOf(Pair("test-field", "test-value1"))), true), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, listOf(Pair("test-field", "test-value2"))), true) + ) + + @JvmStatic + fun multipleFiltersMatch(): List = listOf( + arguments(simpleMessageBuilder("test", Direction.FIRST, "test-alias"), true), + arguments(simpleMessageBuilder("test", Direction.SECOND, "test-alias"), true), + arguments(simpleMessageBuilder("test-wrong", Direction.FIRST, "test-alias"), true), + arguments(simpleMessageBuilder("test-wrong", Direction.SECOND, "test-alias"), false) + ) + + @JvmStatic + fun messagesWithMessageAndMetadataFilters() : List = listOf( + // fields full match + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value2"))), true), + + // metadata mismatch + arguments(messageWithFieldsBuilder("test", Direction.SECOND, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value2"))), false), + arguments(messageWithFieldsBuilder("test-wrong", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value2"))), false), + + // fields mismatch + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value2"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value-wrong"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value-wrong"))), false), + + // one field and one metadata mismatch + arguments(messageWithFieldsBuilder("test", Direction.SECOND, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value2"))), false), + arguments(messageWithFieldsBuilder("test-wrong", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value-wrong"))), false) + ) + + @JvmStatic + fun parsedMessagesBothFilters() : List = listOf( + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value2"))), true), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value2"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value-wrong"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value-wrong"))), false) + ) + + @JvmStatic + fun messagesWithOneProperty() : List = listOf( + arguments(messageWithOnePropertyBuilder("test", "test-property", "property-value"), true), + arguments(messageWithOnePropertyBuilder("test", "test-property", "property-value-wrong"), false), + arguments(rawMessageWithOnePropertyBuilder("test-property", "property-value"), true), + arguments(rawMessageWithOnePropertyBuilder("test-property", "property-value-wrong"), false) + ) + + @JvmStatic + fun messagesWithPropertiesAndMetadata() : List = listOf( + arguments(messageWithOnePropertyBuilder("test", "test-property", "property-value"), true), + arguments(messageWithOnePropertyBuilder("test", "test-property", "property-value-wrong"), false), + arguments(messageWithOnePropertyBuilder("test-wrong", "test-property", "property-value"), false) + ) + + @JvmStatic + fun rawMessagesBothFilters() : List = listOf( + arguments(simpleRawMessageBuilder("test-alias", Direction.FIRST), true), + arguments(simpleRawMessageBuilder("test-alias", Direction.SECOND), false), + arguments(simpleRawMessageBuilder("test-alias-wrong-value", Direction.FIRST), false), + arguments(simpleRawMessageBuilder("test-alias-wrong-value", Direction.SECOND), false) + ) } } \ No newline at end of file From 5afa969ae3b8cfe7196c3797bf1e7854744f9269 Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:50:15 +0400 Subject: [PATCH 127/154] Bump rebuild From 9c9ebb9f4473c4914352cb37c2de430b04c06992 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Sat, 12 Aug 2023 11:40:16 +0400 Subject: [PATCH 128/154] [TH2-4887] Used release dependencies --- build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index a1a03c920..ebb15bb31 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ group = 'com.exactpro.th2' version = release_version ext { - cradleVersion = '5.1.0-separate-executor-5334417446-8a1a37d-SNAPSHOT' + cradleVersion = '5.1.1-dev' junitVersion = '5.9.3' } @@ -167,8 +167,8 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:4.4.0-update-libs-5335052162-8affa4a-SNAPSHOT") - api('com.exactpro.th2:grpc-common:4.3.0-TH2-4887-vulnerabilities-5346500826-f180a19-SNAPSHOT') { + api platform("com.exactpro.th2:bom:4.4.0") + api('com.exactpro.th2:grpc-common:4.3.0-dev') { because('protobuf transport is main now, this dependnecy should be moved to grpc, mq protobuf modules after splitting') } api("com.exactpro.th2:cradle-core:${cradleVersion}") { @@ -182,7 +182,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-generator-annprocess:0.9' implementation 'com.google.protobuf:protobuf-java-util' - implementation 'com.exactpro.th2:grpc-service-generator:3.4.0-TH2-4887-vulnerabilities-5346040251-1bed6cd-SNAPSHOT' + implementation 'com.exactpro.th2:grpc-service-generator:3.4.0' implementation "com.exactpro.th2:cradle-cassandra:${cradleVersion}" def autoValueVersion = '1.10.1' From 43362e8eb3cd4478a2ee03e45309fbcf754a39e0 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Sat, 12 Aug 2023 11:41:52 +0400 Subject: [PATCH 129/154] [TH2-4887] Remove dev-version branch from ignore pattern --- .github/workflows/dev-java-publish-sonatype.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dev-java-publish-sonatype.yml b/.github/workflows/dev-java-publish-sonatype.yml index 488934c05..ec39beb2f 100644 --- a/.github/workflows/dev-java-publish-sonatype.yml +++ b/.github/workflows/dev-java-publish-sonatype.yml @@ -5,7 +5,6 @@ on: branches-ignore: - master - version-* - - dev-version-* - dependabot* # paths: # - gradle.properties From dd0d1ee53ba925ce24db26e05ffe79e09ee6be9d Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Sat, 12 Aug 2023 12:04:02 +0400 Subject: [PATCH 130/154] [TH2-4887] Updated gradle plugins --- Dockerfile | 2 +- build.gradle | 31 ++++++++++++++++++++++-- gradle/wrapper/gradle-wrapper.properties | 2 +- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c7e2c2e67..aa3658bda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM gradle:7.5.1-jdk11 AS build +FROM gradle:7.6-jdk11 AS build ARG release_version ARG bintray_user ARG bintray_key diff --git a/build.gradle b/build.gradle index ebb15bb31..90ad2d658 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,6 @@ +import com.github.jk1.license.filter.LicenseBundleNormalizer +import com.github.jk1.license.render.JsonReportRenderer + buildscript { repositories { mavenCentral() @@ -22,8 +25,11 @@ plugins { id 'com.google.protobuf' version '0.8.8' apply false id 'org.jetbrains.kotlin.jvm' version "${kotlin_version}" id 'org.jetbrains.kotlin.kapt' version "${kotlin_version}" - id "org.owasp.dependencycheck" version "8.2.1" + id "org.owasp.dependencycheck" version "8.3.1" id "me.champeau.jmh" version "0.6.8" + id "com.gorylenko.gradle-git-properties" version "2.4.1" + id 'com.github.jk1.dependency-license-report' version '2.5' + id "de.undercouch.download" version "5.4.0" } group = 'com.exactpro.th2' @@ -285,4 +291,25 @@ dependencyCheck { nugetconfEnabled = false nodeEnabled = false } -} \ No newline at end of file +} + +licenseReport { + def licenseNormalizerBundlePath = "$buildDir/license-normalizer-bundle.json" + + if (!file(licenseNormalizerBundlePath).exists()) { + download.run { + src 'https://raw.githubusercontent.com/th2-net/.github/main/license-compliance/gradle-license-report/license-normalizer-bundle.json' + dest "$buildDir/license-normalizer-bundle.json" + overwrite false + } + } + + filters = [ + new LicenseBundleNormalizer(licenseNormalizerBundlePath, false) + ] + renderers = [ + new JsonReportRenderer('licenses.json', false), + ] + excludeOwnGroup = false + allowedLicensesFile = new URL("https://raw.githubusercontent.com/th2-net/.github/main/license-compliance/gradle-license-report/allowed-licenses.json") +} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 7d1733249..8c6e09b97 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ #Thu Jul 02 11:31:27 GMT+04:00 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists From 6376242837e2ef53ca6ff0dc0e5528dbae1f2497 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Sat, 12 Aug 2023 12:09:03 +0400 Subject: [PATCH 131/154] [TH2-4887] Updated README * Updated junit-jupiter:5.10.0 --- README.md | 12 ++++++++++++ build.gradle | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 163f18e4a..75e6807c0 100644 --- a/README.md +++ b/README.md @@ -459,6 +459,18 @@ dependencies { + Implemented message routers used th2 transport protocol for interaction +#### Updated: ++ cradle: `5.1.1-dev` ++ bom: `4.4.0` ++ grpc-common: `4.3.0-dev` ++ grpc-service-generator: `3.4.0` + +#### Gradle plugins: ++ Updated org.owasp.dependencycheck: `8.3.1` ++ Added com.gorylenko.gradle-git-properties `2.4.1` ++ Added com.github.jk1.dependency-license-report `2.5` ++ Added de.undercouch.download `5.4.0` + ### 5.2.1 #### Changed: diff --git a/build.gradle b/build.gradle index 90ad2d658..f7bf74f55 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ version = release_version ext { cradleVersion = '5.1.1-dev' - junitVersion = '5.9.3' + junitVersion = '5.10.0' } repositories { From 69665013512f84c2294c9d007fcedfad91f01292 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 12 Jul 2023 10:55:10 +0400 Subject: [PATCH 132/154] [TH2-4887] Replaced JsonProcessingException to IOException in the Event class --- .../com/exactpro/th2/common/event/Event.java | 119 ++++++++++-------- 1 file changed, 66 insertions(+), 53 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/event/Event.java b/src/main/java/com/exactpro/th2/common/event/Event.java index b69b5a057..d4c291c49 100644 --- a/src/main/java/com/exactpro/th2/common/event/Event.java +++ b/src/main/java/com/exactpro/th2/common/event/Event.java @@ -15,6 +15,29 @@ */ package com.exactpro.th2.common.event; +import com.exactpro.th2.common.grpc.EventBatch; +import com.exactpro.th2.common.grpc.EventBatchOrBuilder; +import com.exactpro.th2.common.grpc.EventID; +import com.exactpro.th2.common.grpc.EventStatus; +import com.exactpro.th2.common.grpc.MessageID; +import com.exactpro.th2.common.message.MessageUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.ByteString; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + import static com.exactpro.th2.common.event.EventUtils.createMessageBean; import static com.exactpro.th2.common.event.EventUtils.generateUUID; import static com.exactpro.th2.common.event.EventUtils.requireNonBlankBookName; @@ -31,30 +54,6 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; - -import com.exactpro.th2.common.message.MessageUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.exactpro.th2.common.grpc.EventBatch; -import com.exactpro.th2.common.grpc.EventBatchOrBuilder; -import com.exactpro.th2.common.grpc.EventID; -import com.exactpro.th2.common.grpc.EventStatus; -import com.exactpro.th2.common.grpc.MessageID; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.ByteString; - //TODO: move to common-utils-j @SuppressWarnings("unused") public class Event { @@ -95,6 +94,7 @@ protected Event() { /** * Creates event with current time as start + * * @return new event */ public static Event start() { @@ -103,6 +103,7 @@ public static Event start() { /** * Creates event with passed time as start + * * @return new event */ public static Event from(@NotNull Instant startTimestamp) { @@ -120,6 +121,7 @@ public Event endTimestamp() { /** * Sets event name if passed {@code eventName} is not blank. * The {@link #UNKNOWN_EVENT_NAME} value will be used as default in the {@link #toProto(com.exactpro.th2.common.grpc.EventID)} and {@link #toListProto(com.exactpro.th2.common.grpc.EventID)} methods if this property isn't set + * * @return current event * @throws IllegalStateException if name already set */ @@ -136,6 +138,7 @@ public Event name(String eventName) { /** * Sets event description if passed {@code description} is not blank. * This property value will be appended to the end of event name and added into event body in the {@link #toProto(com.exactpro.th2.common.grpc.EventID)} and {@link #toListProto(com.exactpro.th2.common.grpc.EventID)} methods if this property isn't set + * * @return current event * @throws IllegalStateException if description already set */ @@ -153,6 +156,7 @@ public Event description(String description) { /** * Sets event type if passed {@code eventType} is not blank. * The {@link #UNKNOWN_EVENT_TYPE} value will be used as default in the {@link #toProto(com.exactpro.th2.common.grpc.EventID)} and {@link #toListProto(com.exactpro.th2.common.grpc.EventID)} methods if this property isn't set + * * @return current event * @throws IllegalStateException if type already set */ @@ -169,6 +173,7 @@ public Event type(String eventType) { /** * Sets event status if passed {@code eventStatus} isn't null. * The default value is {@link Status#PASSED} + * * @return current event */ public Event status(Status eventStatus) { @@ -180,6 +185,7 @@ public Event status(Status eventStatus) { /** * Creates and adds a new event with the same start / end time as the current event + * * @return created event */ @SuppressWarnings("NonBooleanMethodNameMayNotStartWithQuestion") @@ -189,6 +195,7 @@ public Event addSubEventWithSamePeriod() { /** * Adds passed event as a sub event + * * @return passed event * @throws NullPointerException if {@code subEvent} is null */ @@ -200,6 +207,7 @@ public Event addSubEvent(Event subEvent) { /** * Adds passed body data bodyData + * * @return current event */ public Event bodyData(IBodyData bodyData) { @@ -209,6 +217,7 @@ public Event bodyData(IBodyData bodyData) { /** * Adds passed collection of body data + * * @return current event */ public Event bodyData(Collection bodyDataCollection) { @@ -218,6 +227,7 @@ public Event bodyData(Collection bodyDataCollection) { /** * Adds the passed exception and optionally all the causes to the body data as a series of messages + * * @param includeCauses if `true` attache messages for the caused of throwable * @return current event */ @@ -232,6 +242,7 @@ public Event exception(@NotNull Throwable throwable, boolean includeCauses) { /** * Adds message id as linked + * * @return current event */ public Event messageID(MessageID attachedMessageID) { @@ -244,7 +255,7 @@ public Event messageID(MessageID attachedMessageID) { return this; } - public List toListProto(@NotNull EventID parentId) throws JsonProcessingException { + public List toListProto(@NotNull EventID parentId) throws IOException { return toListProto( new ArrayList<>(), requireNonNullParentId(parentId), @@ -253,7 +264,7 @@ public List toListProto(@NotNull EventID par ); } - public List toListProto(@NotNull String bookName) throws JsonProcessingException { + public List toListProto(@NotNull String bookName) throws IOException { return toListProto( new ArrayList<>(), null, @@ -265,7 +276,7 @@ public List toListProto(@NotNull String book public List toListProto( @NotNull String bookName, @NotNull String scope - ) throws JsonProcessingException { + ) throws IOException { return toListProto( new ArrayList<>(), null, @@ -279,7 +290,7 @@ private List toListProto( @Nullable EventID parentId, @NotNull String bookName, @Nullable String scope - ) throws JsonProcessingException { + ) throws IOException { protoEvents.add(toProto(parentId, bookName, scope)); // collect current level for (Event subEvent : subEvents) { EventID eventId = isBlank(scope) @@ -290,7 +301,7 @@ private List toListProto( return protoEvents; } - public com.exactpro.th2.common.grpc.Event toProto(@NotNull EventID parentId) throws JsonProcessingException { + public com.exactpro.th2.common.grpc.Event toProto(@NotNull EventID parentId) throws IOException { return toProto( requireNonNullParentId(parentId), requireNonBlankBookName(parentId.getBookName()), @@ -298,7 +309,7 @@ public com.exactpro.th2.common.grpc.Event toProto(@NotNull EventID parentId) thr ); } - public com.exactpro.th2.common.grpc.Event toProto(@NotNull String bookName) throws JsonProcessingException { + public com.exactpro.th2.common.grpc.Event toProto(@NotNull String bookName) throws IOException { return toProto( null, requireNonBlankBookName(bookName), @@ -309,7 +320,7 @@ public com.exactpro.th2.common.grpc.Event toProto(@NotNull String bookName) thro public com.exactpro.th2.common.grpc.Event toProto( @NotNull String bookName, @Nullable String scope - ) throws JsonProcessingException { + ) throws IOException { return toProto( null, requireNonBlankBookName(bookName), @@ -321,7 +332,7 @@ private com.exactpro.th2.common.grpc.Event toProto( @Nullable EventID parentId, @NotNull String bookName, @Nullable String scope - ) throws JsonProcessingException { + ) throws IOException { if (endTimestamp == null) { endTimestamp(); } @@ -349,7 +360,7 @@ private com.exactpro.th2.common.grpc.Event toProto( return eventBuilder.build(); } - public EventBatch toBatchProto(@NotNull EventID parentId) throws JsonProcessingException { + public EventBatch toBatchProto(@NotNull EventID parentId) throws IOException { return toBatchProto( requireNonNullParentId(parentId), requireNonBlankBookName(parentId.getBookName()), @@ -357,7 +368,7 @@ public EventBatch toBatchProto(@NotNull EventID parentId) throws JsonProcessingE ); } - public EventBatch toBatchProto(@NotNull String bookName) throws JsonProcessingException { + public EventBatch toBatchProto(@NotNull String bookName) throws IOException { return toBatchProto( null, requireNonBlankBookName(bookName), @@ -368,7 +379,7 @@ public EventBatch toBatchProto(@NotNull String bookName) throws JsonProcessingEx public EventBatch toBatchProto( @NotNull String bookName, @NotNull String scope - ) throws JsonProcessingException { + ) throws IOException { return toBatchProto( null, requireNonBlankBookName(bookName), @@ -380,11 +391,11 @@ private EventBatch toBatchProto( @Nullable EventID parentId, @NotNull String bookName, @Nullable String scope - ) throws JsonProcessingException { + ) throws IOException { List events = toListProto(new ArrayList<>(), parentId, bookName, scope); EventBatch.Builder builder = EventBatch.newBuilder() - .addAllEvents(events); + .addAllEvents(events); if (parentId != null && events.size() != 1) { builder.setParentEventId(parentId); @@ -396,7 +407,7 @@ private EventBatch toBatchProto( public List toBatchesProtoWithLimit( int maxEventBatchContentSize, @NotNull EventID parentId - ) throws JsonProcessingException { + ) throws IOException { return toBatchesProtoWithLimit( maxEventBatchContentSize, requireNonNullParentId(parentId), @@ -408,7 +419,7 @@ public List toBatchesProtoWithLimit( public List toBatchesProtoWithLimit( int maxEventBatchContentSize, @NotNull String bookName - ) throws JsonProcessingException { + ) throws IOException { return toBatchesProtoWithLimit( maxEventBatchContentSize, null, @@ -421,7 +432,7 @@ public List toBatchesProtoWithLimit( int maxEventBatchContentSize, @NotNull String bookName, @NotNull String scope - ) throws JsonProcessingException { + ) throws IOException { return toBatchesProtoWithLimit( maxEventBatchContentSize, null, @@ -435,17 +446,18 @@ public List toBatchesProtoWithLimit( * Splitting to batch executes by principles: * * Events with children are put into distinct batches because events can't be a child of an event from another batch. * * Events without children are collected into batches according to the max size. For example, little child events can be put into one batch; big child events can be put into separate batches. + * * @param maxEventBatchContentSize - the maximum size of useful content in one batch which is calculated as the sum of the size of all event bodies in the batch * @param parentId - reference to parent event for the current event tree. It may be null if the current event is root, in this case {@code bookName} is required. * @param bookName - book name for the current event tree. It may not be null. * @param scope - scope for the current event tree. It may be null. */ private List toBatchesProtoWithLimit( - int maxEventBatchContentSize, + int maxEventBatchContentSize, @Nullable EventID parentId, @NotNull String bookName, @Nullable String scope - ) throws JsonProcessingException { + ) throws IOException { if (maxEventBatchContentSize <= 0) { throw new IllegalArgumentException("'maxEventBatchContentSize' should be greater than zero, actual: " + maxEventBatchContentSize); } @@ -458,7 +470,7 @@ private List toBatchesProtoWithLimit( return result; } - public List toListBatchProto(@NotNull EventID parentId) throws JsonProcessingException { + public List toListBatchProto(@NotNull EventID parentId) throws IOException { return toListBatchProto( requireNonNullParentId(parentId), requireNonBlankBookName(parentId.getBookName()), @@ -466,7 +478,7 @@ public List toListBatchProto(@NotNull EventID parentId) throws JsonP ); } - public List toListBatchProto(@NotNull String bookName) throws JsonProcessingException { + public List toListBatchProto(@NotNull String bookName) throws IOException { return toListBatchProto( null, requireNonBlankBookName(bookName), @@ -477,7 +489,7 @@ public List toListBatchProto(@NotNull String bookName) throws JsonPr public List toListBatchProto( @NotNull String bookName, @NotNull String scope - ) throws JsonProcessingException { + ) throws IOException { return toListBatchProto( null, requireNonBlankBookName(bookName), @@ -490,6 +502,7 @@ public List toListBatchProto( * Splitting to batch executes by principles: * * Events with children are put into distinct batches because events can't be a child of an event from another batch. * * Events without children are collected into batches. + * * @param parentId - reference to parent event for the current event tree. It may be null if the current event is root, in this case {@code bookName} is required. * @param bookName - book name for the current event tree. It may not be null. * @param scope - scope for the current event tree. It may be null. @@ -498,7 +511,7 @@ private List toListBatchProto( @Nullable EventID parentId, @NotNull String bookName, @Nullable String scope - ) throws JsonProcessingException { + ) throws IOException { return toBatchesProtoWithLimit(Integer.MAX_VALUE, parentId, bookName, scope); } @@ -514,7 +527,7 @@ public Instant getEndTimestamp() { return endTimestamp; } - protected byte[] buildBody() throws JsonProcessingException { + protected byte[] buildBody() throws IOException { return OBJECT_MAPPER.get().writeValueAsBytes(body); } @@ -532,7 +545,7 @@ protected Status getAggregatedStatus() { return Status.FAILED; } - private void batch(int maxEventBatchContentSize, List result, Map> eventGroups, @Nullable EventID eventID) throws JsonProcessingException { + private void batch(int maxEventBatchContentSize, List result, Map> eventGroups, @Nullable EventID eventID) throws IOException { eventID = requireNonNullElse(eventID, DEFAULT_EVENT_ID); EventBatch.Builder builder = setParentId(EventBatch.newBuilder(), eventID); @@ -545,13 +558,13 @@ private void batch(int maxEventBatchContentSize, List result, Map 0 + if (builder.getEventsCount() > 0 && getContentSize(builder) + getContentSize(checkedProtoEvent) > maxEventBatchContentSize) { result.add(checkAndBuild(maxEventBatchContentSize, builder)); builder = setParentId(EventBatch.newBuilder(), eventID); @@ -560,21 +573,21 @@ && getContentSize(builder) + getContentSize(checkedProtoEvent) > maxEventBatchCo } } - if(builder.getEventsCount() > 0) { + if (builder.getEventsCount() > 0) { result.add(checkAndBuild(maxEventBatchContentSize, builder)); } } private EventBatch checkAndBuild(int maxEventBatchContentSize, EventBatch.Builder builder) { int contentSize = getContentSize(builder); - if(contentSize > maxEventBatchContentSize) { + if (contentSize > maxEventBatchContentSize) { throw new IllegalStateException("The smallest batch size exceeds the max event batch content size, max " + maxEventBatchContentSize + ", actual " + contentSize); } return builder.build(); } - private com.exactpro.th2.common.grpc.Event checkAndRebuild(int maxEventBatchContentSize, com.exactpro.th2.common.grpc.Event event) throws JsonProcessingException { + private com.exactpro.th2.common.grpc.Event checkAndRebuild(int maxEventBatchContentSize, com.exactpro.th2.common.grpc.Event event) throws IOException { int contentSize = getContentSize(event); if (contentSize > maxEventBatchContentSize) { return com.exactpro.th2.common.grpc.Event.newBuilder(event) From 004806a1437a01d2c0cd84861d83581b59bc3a01 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 12 Jul 2023 10:55:10 +0400 Subject: [PATCH 133/154] [TH2-4887] fixed compile problem --- .../schema/factory/AbstractCommonFactory.java | 111 ++++++++++-------- 1 file changed, 65 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 48cf769f7..a6f3d2000 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -58,7 +58,6 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportGroupBatchRouter; import com.exactpro.th2.common.schema.strategy.route.json.RoutingStrategyModule; import com.exactpro.th2.common.schema.util.Log4jConfigUtils; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fasterxml.jackson.module.kotlin.KotlinFeature; @@ -103,7 +102,6 @@ import static org.apache.commons.lang3.StringUtils.defaultIfBlank; /** - * * Class for load JSON schema configuration and create {@link GrpcRouter} and {@link MessageRouter} * * @see CommonFactory @@ -112,7 +110,9 @@ public abstract class AbstractCommonFactory implements AutoCloseable { protected static final String EXACTPRO_IMPLEMENTATION_VENDOR = "Exactpro Systems LLC"; - /** @deprecated please use {@link #LOG4J_PROPERTIES_DEFAULT_PATH} */ + /** + * @deprecated please use {@link #LOG4J_PROPERTIES_DEFAULT_PATH} + */ @Deprecated protected static final String LOG4J_PROPERTIES_DEFAULT_PATH_OLD = "/home/etc"; protected static final String LOG4J_PROPERTIES_DEFAULT_PATH = "/var/th2/config"; @@ -120,7 +120,7 @@ public abstract class AbstractCommonFactory implements AutoCloseable { public static final ObjectMapper MAPPER = new ObjectMapper(); - static { + static { MAPPER.registerModules( new KotlinModule.Builder() .withReflectionCacheSize(512) @@ -165,6 +165,7 @@ public abstract class AbstractCommonFactory implements AutoCloseable { /** * Create factory with non-default implementations schema classes + * * @param settings {@link FactorySettings} */ public AbstractCommonFactory(FactorySettings settings) { @@ -179,6 +180,7 @@ public AbstractCommonFactory(FactorySettings settings) { /** * Create factory with default implementation schema classes + * * @deprecated Please use {@link AbstractCommonFactory#AbstractCommonFactory(FactorySettings)} */ @Deprecated(since = "4.0.0", forRemoval = true) @@ -188,6 +190,7 @@ public AbstractCommonFactory() { /** * Create factory with non-default implementations schema classes + * * @param messageRouterParsedBatchClass Class for {@link MessageRouter} which work with {@link MessageBatch} * @param messageRouterRawBatchClass Class for {@link MessageRouter} which work with {@link RawMessageBatch} * @param eventBatchRouterClass Class for {@link MessageRouter} which work with {@link EventBatch} @@ -195,11 +198,13 @@ public AbstractCommonFactory() { * @deprecated Please use {@link AbstractCommonFactory#AbstractCommonFactory(FactorySettings)} */ @Deprecated(since = "4.0.0", forRemoval = true) - public AbstractCommonFactory(@NotNull Class> messageRouterParsedBatchClass, + public AbstractCommonFactory( + @NotNull Class> messageRouterParsedBatchClass, @NotNull Class> messageRouterRawBatchClass, @NotNull Class> messageRouterMessageGroupBatchClass, @NotNull Class> eventBatchRouterClass, - @NotNull Class grpcRouterClass) { + @NotNull Class grpcRouterClass + ) { this(new FactorySettings() .messageRouterParsedBatchClass(messageRouterParsedBatchClass) .messageRouterRawBatchClass(messageRouterRawBatchClass) @@ -220,12 +225,14 @@ public AbstractCommonFactory(@NotNull Class> messageRouterParsedBatchClass, + protected AbstractCommonFactory( + @NotNull Class> messageRouterParsedBatchClass, @NotNull Class> messageRouterRawBatchClass, @NotNull Class> messageRouterMessageGroupBatchClass, @NotNull Class> eventBatchRouterClass, @NotNull Class grpcRouterClass, - @NotNull Map environmentVariables) { + @NotNull Map environmentVariables + ) { this(new FactorySettings() .messageRouterParsedBatchClass(messageRouterParsedBatchClass) .messageRouterRawBatchClass(messageRouterRawBatchClass) @@ -265,7 +272,8 @@ public MessageRouter getMessageRouterParsedBatch() { try { router = messageRouterParsedBatchClass.getConstructor().newInstance(); router.init(getMessageRouterContext(), getMessageRouterMessageGroupBatch()); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { throw new CommonFactoryException("Can not create parsed message router", e); } } @@ -285,7 +293,8 @@ public MessageRouter getMessageRouterRawBatch() { try { router = messageRouterRawBatchClass.getConstructor().newInstance(); router.init(getMessageRouterContext(), getMessageRouterMessageGroupBatch()); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { throw new CommonFactoryException("Can not create raw message router", e); } } @@ -320,7 +329,8 @@ public MessageRouter getMessageRouterMessageGroupBatch() { try { router = messageRouterMessageGroupBatchClass.getConstructor().newInstance(); router.init(getMessageRouterContext()); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { throw new CommonFactoryException("Can not create group message router", e); } } @@ -345,7 +355,8 @@ public MessageRouter getEventBatchRouter() { getMessageRouterConfiguration(), getBoxConfiguration() )); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { throw new CommonFactoryException("Can not create event batch router", e); } } @@ -364,7 +375,8 @@ public GrpcRouter getGrpcRouter() { try { router = grpcRouterClass.getConstructor().newInstance(); router.init(getGrpcConfiguration(), getGrpcRouterConfiguration()); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { throw new CommonFactoryException("Can not create GRPC router", e); } } @@ -384,7 +396,8 @@ public NotificationRouter getNotificationEventBatchRouter() { try { router = notificationEventBatchRouterClass.getConstructor().newInstance(); router.init(getMessageRouterContext()); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { throw new CommonFactoryException("Can not create notification router", e); } } @@ -409,11 +422,11 @@ public void registerCustomMessageRouter( /** * Registers message router for custom type that is passed via {@code messageClass} parameter.
* - * @param messageClass custom message class - * @param messageConverter converter that will be used to convert message to bytes and vice versa - * @param defaultSendAttributes set of attributes for sending. A pin must have all of them to be selected for sending the message + * @param messageClass custom message class + * @param messageConverter converter that will be used to convert message to bytes and vice versa + * @param defaultSendAttributes set of attributes for sending. A pin must have all of them to be selected for sending the message * @param defaultSubscribeAttributes set of attributes for subscription. A pin must have all of them to be selected for receiving messages - * @param custom message type + * @param custom message type * @throws IllegalStateException if the router for {@code messageClass} is already registered */ public void registerCustomMessageRouter( @@ -441,10 +454,11 @@ public void registerCustomMessageRouter( /** * Returns previously registered message router for message of {@code messageClass} type. * If the router for that type is not registered yet ,it throws {@link IllegalArgumentException} + * * @param messageClass custom message class - * @param custom message type - * @throws IllegalArgumentException if router for specified type is not registered + * @param custom message type * @return the previously registered router for specified type + * @throws IllegalArgumentException if router for specified type is not registered */ @SuppressWarnings("unchecked") @NotNull @@ -454,7 +468,7 @@ public MessageRouter getCustomMessageRouter(Class messageClass) { throw new IllegalArgumentException( "Router for class " + messageClass.getCanonicalName() + "is not registered. Call 'registerCustomMessageRouter' first"); } - return (MessageRouter)router; + return (MessageRouter) router; } /** @@ -467,8 +481,9 @@ public T getConfiguration(Path configPath, Class configClass, ObjectMappe /** * Load configuration, save and return. If already loaded return saved configuration. + * * @param configClass configuration class - * @param optional creates an instance of a configuration class via the default constructor if this option is true and the config file doesn't exist or empty + * @param optional creates an instance of a configuration class via the default constructor if this option is true and the config file doesn't exist or empty * @return configuration object */ protected T getConfigurationOrLoad(Class configClass, boolean optional) { @@ -582,7 +597,8 @@ public T getCustomConfiguration(Class confClass, ObjectMapper customObjec if (!configFile.exists()) { try { return confClass.getConstructor().newInstance(); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { return null; } } @@ -604,6 +620,7 @@ public T getCustomConfiguration(Class confClass) { /** * Read first and only one dictionary + * * @return Dictionary as {@link InputStream} * @throws IllegalStateException if can not read dictionary or found more than one target */ @@ -624,6 +641,7 @@ public T getCustomConfiguration(Class confClass) { /** * Read dictionary of {@link DictionaryType#MAIN} type + * * @return Dictionary as {@link InputStream} * @throws IllegalStateException if can not read dictionary */ @@ -631,16 +649,17 @@ public T getCustomConfiguration(Class confClass) { public abstract InputStream readDictionary(); /** - * @deprecated Dictionary types will be removed in future releases of infra, use alias instead * @param dictionaryType desired type of dictionary * @return Dictionary as {@link InputStream} * @throws IllegalStateException if can not read dictionary + * @deprecated Dictionary types will be removed in future releases of infra, use alias instead */ @Deprecated(since = "3.33.0", forRemoval = true) public abstract InputStream readDictionary(DictionaryType dictionaryType); /** * If root event does not exist, it creates root event with its book name = box book name and name = box name and timestamp + * * @return root event id */ @NotNull @@ -664,7 +683,7 @@ public EventID getRootEventId() { } catch (IOException e) { throw new CommonFactoryException("Can not send root event", e); } - } catch (JsonProcessingException e) { + } catch (IOException e) { throw new CommonFactoryException("Can not create root event", e); } } @@ -698,27 +717,27 @@ public EventID getRootEventId() { */ protected MessageRouterContext getMessageRouterContext() { return routerContext.updateAndGet(ctx -> { - if (ctx == null) { - try { - MessageRouterMonitor contextMonitor = new BroadcastMessageRouterMonitor( - new LogMessageRouterMonitor(), - new EventMessageRouterMonitor( - getEventBatchRouter(), - getRootEventId() - ) - ); - - return new DefaultMessageRouterContext( - getRabbitMqConnectionManager(), - contextMonitor, - getMessageRouterConfiguration(), - getBoxConfiguration() - ); - } catch (Exception e) { - throw new CommonFactoryException("Can not create message router context", e); - } - } - return ctx; + if (ctx == null) { + try { + MessageRouterMonitor contextMonitor = new BroadcastMessageRouterMonitor( + new LogMessageRouterMonitor(), + new EventMessageRouterMonitor( + getEventBatchRouter(), + getRootEventId() + ) + ); + + return new DefaultMessageRouterContext( + getRabbitMqConnectionManager(), + contextMonitor, + getMessageRouterConfiguration(), + getBoxConfiguration() + ); + } catch (Exception e) { + throw new CommonFactoryException("Can not create message router context", e); + } + } + return ctx; }); } From fb4b460a2dd7a4f7dc690fa569832021a4549e41 Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Thu, 13 Jul 2023 10:54:22 +0000 Subject: [PATCH 134/154] [TH2-4987] added com.gorylenko.gradle-git-properties plugin (#269) --- .../schema/factory/AbstractCommonFactory.java | 34 +----- .../common/schema/factory/ExactproMetaInf.kt | 115 ++++++++++++++++++ 2 files changed, 116 insertions(+), 33 deletions(-) create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/factory/ExactproMetaInf.kt diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index a6f3d2000..d316c5129 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -74,24 +74,16 @@ import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; -import java.net.URL; import java.nio.file.Path; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.Spliterators; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; -import java.util.jar.Attributes.Name; -import java.util.jar.JarFile; -import java.util.jar.Manifest; -import java.util.stream.StreamSupport; import static com.exactpro.cradle.CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE; import static com.exactpro.cradle.CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE; @@ -108,8 +100,6 @@ */ public abstract class AbstractCommonFactory implements AutoCloseable { - protected static final String EXACTPRO_IMPLEMENTATION_VENDOR = "Exactpro Systems LLC"; - /** * @deprecated please use {@link #LOG4J_PROPERTIES_DEFAULT_PATH} */ @@ -872,28 +862,6 @@ protected static void configureLogger(String... paths) { listPath.addAll(Arrays.asList(requireNonNull(paths, "Paths can't be null"))); Log4jConfigUtils log4jConfigUtils = new Log4jConfigUtils(); log4jConfigUtils.configure(listPath, LOG4J2_PROPERTIES_NAME); - loggingManifests(); - } - - private static void loggingManifests() { - try { - Iterator urlIterator = Thread.currentThread().getContextClassLoader().getResources(JarFile.MANIFEST_NAME).asIterator(); - StreamSupport.stream(Spliterators.spliteratorUnknownSize(urlIterator, 0), false) - .map(url -> { - try (InputStream inputStream = url.openStream()) { - return new Manifest(inputStream); - } catch (IOException e) { - LOGGER.warn("Manifest '{}' loading failere", url, e); - return null; - } - }) - .filter(Objects::nonNull) - .map(Manifest::getMainAttributes) - .filter(attributes -> EXACTPRO_IMPLEMENTATION_VENDOR.equals(attributes.getValue(Name.IMPLEMENTATION_VENDOR))) - .forEach(attributes -> LOGGER.info("Manifest title {}, version {}" - , attributes.getValue(Name.IMPLEMENTATION_TITLE), attributes.getValue(Name.IMPLEMENTATION_VERSION))); - } catch (IOException e) { - LOGGER.warn("Manifest searching failure", e); - } + ExactproMetaInf.logging(); } } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/factory/ExactproMetaInf.kt b/src/main/kotlin/com/exactpro/th2/common/schema/factory/ExactproMetaInf.kt new file mode 100644 index 000000000..0295e3a23 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/factory/ExactproMetaInf.kt @@ -0,0 +1,115 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.factory + +import mu.KotlinLogging +import java.io.IOException +import java.net.URL +import java.nio.file.Path +import java.util.Properties +import java.util.jar.Attributes +import java.util.jar.JarFile +import java.util.jar.Manifest + +internal class ExactproMetaInf( + url: URL, + private val title: String, + private val version: String +) { + private val jarPath = Path.of(url.path).parent.parent + + private var gitEnriched = false + private var gitHash = "" + private var gitBranch = "" + private var gitRemoteUrl = "" + private var gitClosestTag = "" + + fun enrich(gitProperty: URL) { + try { + gitProperty.openStream().use { inputStream -> + val properties = Properties() + properties.load(inputStream) + gitHash = properties.getProperty(GIT_HASH_PROPERTY) + gitBranch = properties.getProperty(GIT_BRANCH_PROPERTY) + gitRemoteUrl = properties.getProperty(GIT_REMOTE_URL_PROPERTY) + gitClosestTag = properties.getProperty(GIT_CLOSEST_TAG_PROPERTY) + gitEnriched = true + } + } catch (e: IOException) { + K_LOGGER.warn(e) { "Git properties '$gitProperty' loading failure" } + } + } + + override fun toString(): String { + return "Manifest title: $title, version: $version , git { ${ + if (gitEnriched) { + "hash: $gitHash, branch: $gitBranch, repository: $gitRemoteUrl, closest tag: $gitClosestTag" + } else { + "'${jarPath.fileName}' jar doesn't contain '$GIT_PROPERTIES_FILE' resource, please use '$GRADLE_GIT_PROPERTIES_PLUGIN' plugin" + } + } }" + } + + companion object { + private val K_LOGGER = KotlinLogging.logger {} + private const val EXACTPRO_IMPLEMENTATION_VENDOR = "Exactpro Systems LLC" + private const val GRADLE_GIT_PROPERTIES_PLUGIN = "com.gorylenko.gradle-git-properties" + private const val GIT_PROPERTIES_FILE = "git.properties" + private const val GIT_HASH_PROPERTY = "git.commit.id" + private const val GIT_BRANCH_PROPERTY = "git.branch" + private const val GIT_REMOTE_URL_PROPERTY = "git.remote.origin.url" + private const val GIT_CLOSEST_TAG_PROPERTY = "git.closest.tag.name" + + @JvmStatic + fun logging() { + if (K_LOGGER.isInfoEnabled) { + try { + val map = Thread.currentThread().contextClassLoader + .getResources(JarFile.MANIFEST_NAME).asSequence() + .mapNotNull(::create) + .map { metaInf -> metaInf.jarPath to metaInf } + .toMap() + + Thread.currentThread().contextClassLoader + .getResources(GIT_PROPERTIES_FILE).asSequence() + .forEach { url -> map[Path.of(url.path).parent]?.enrich(url) } + + map.values.forEach { metaInf -> K_LOGGER.info { "$metaInf" } } + } catch (e: IOException) { + K_LOGGER.warn(e) { "Manifest searching failure" } + } + } + } + + private fun create(manifestUrl: URL): ExactproMetaInf? { + try { + manifestUrl.openStream().use { inputStream -> + val attributes = Manifest(inputStream).mainAttributes + return if (EXACTPRO_IMPLEMENTATION_VENDOR != attributes.getValue(Attributes.Name.IMPLEMENTATION_VENDOR)) { + null + } else ExactproMetaInf( + manifestUrl, + attributes.getValue(Attributes.Name.IMPLEMENTATION_TITLE), + attributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION) + ) + } + } catch (e: IOException) { + K_LOGGER.warn(e) { "Manifest '$manifestUrl' loading failure" } + return null + } + } + } +} From d0e80343c5e9ab1b084ab1e04a1cb7245a6d9124 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Sat, 12 Aug 2023 12:36:04 +0400 Subject: [PATCH 135/154] [TH2-4887] Updated README --- README.md | 13 +++++++++++-- gradle.properties | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 75e6807c0..599df2e36 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (5.3.0) +# th2 common library (Java) (5.3.1) ## Usage @@ -455,7 +455,16 @@ dependencies { ## Release notes -### 5.3.0 +### 5.3.1-dev + ++ Auto-print git metadata from `git.properties` resource file. + Child project should include `com.gorylenko.gradle-git-properties` Gradle plugin to generate required file + +#### Change user code required: ++ Migrated from JsonProcessingException to IOException in Event class methods. + This change allow remove required `jackson.core` dependency from child projects + +### 5.3.0-dev + Implemented message routers used th2 transport protocol for interaction diff --git a/gradle.properties b/gradle.properties index b8ef7806b..4b6b65a46 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.3.0 +release_version=5.3.1 description='th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j kapt.include.compile.classpath=false From 43fc21564db83aa0898cca23f6c30649434c104a Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 17 Jul 2023 19:08:58 +0400 Subject: [PATCH 136/154] [TH2-4989] Added equals, hashCode and toBuilder methods for transport classes --- .../impl/rabbitmq/transport/EventId.kt | 27 ++++++++++++++ .../impl/rabbitmq/transport/GroupBatch.kt | 25 +++++++++++++ .../impl/rabbitmq/transport/MessageGroup.kt | 21 +++++++++++ .../impl/rabbitmq/transport/MessageId.kt | 33 +++++++++++++++-- .../impl/rabbitmq/transport/ParsedMessage.kt | 15 ++++++++ .../impl/rabbitmq/transport/RawMessage.kt | 35 ++++++++++++++++++- 6 files changed, 152 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt index 56508e31b..152227d45 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt @@ -25,6 +25,31 @@ data class EventId( val scope: String, val timestamp: Instant, ) { + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as EventId + + if (id != other.id) return false + if (book != other.book) return false + if (scope != other.scope) return false + return timestamp == other.timestamp + } + + override fun hashCode(): Int { + var result = id.hashCode() + result = 31 * result + book.hashCode() + result = 31 * result + scope.hashCode() + result = 31 * result + timestamp.hashCode() + return result + } + + override fun toString(): String { + return "EventId(id='$id', book='$book', scope='$scope', timestamp=$timestamp)" + } + @AutoBuilder interface Builder { val id: String @@ -39,6 +64,8 @@ data class EventId( fun build(): EventId } + fun toBuilder(): Builder = AutoBuilder_EventId_Builder(this) + companion object { @JvmStatic fun builder(): Builder = AutoBuilder_EventId_Builder() diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt index 74e98c6f3..933e78d14 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/GroupBatch.kt @@ -24,6 +24,29 @@ data class GroupBatch( val sessionGroup: String, val groups: List = emptyList(), ) { + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as GroupBatch + + if (book != other.book) return false + if (sessionGroup != other.sessionGroup) return false + return groups == other.groups + } + + override fun hashCode(): Int { + var result = book.hashCode() + result = 31 * result + sessionGroup.hashCode() + result = 31 * result + groups.hashCode() + return result + } + + override fun toString(): String { + return "GroupBatch(book='$book', sessionGroup='$sessionGroup', groups=$groups)" + } + @AutoBuilder interface Builder { val book: String @@ -40,6 +63,8 @@ data class GroupBatch( fun build(): GroupBatch } + fun toBuilder(): Builder = AutoBuilder_GroupBatch_Builder(this) + companion object { @JvmStatic fun builder(): Builder = AutoBuilder_GroupBatch_Builder() diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt index 3756e548b..00bbd1f74 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageGroup.kt @@ -23,6 +23,23 @@ import com.google.auto.value.AutoBuilder data class MessageGroup( val messages: List> = emptyList(), // FIXME: message can have incompatible book and group ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as MessageGroup + + return messages == other.messages + } + + override fun hashCode(): Int { + return messages.hashCode() + } + + override fun toString(): String { + return "MessageGroup(messages=$messages)" + } + // We cannot use AutoBuilder here because of the different method signatures in builder when a generic type is used // @AutoBuilder @@ -31,10 +48,14 @@ data class MessageGroup( fun addMessage(message: Message<*>): Builder = apply { messagesBuilder().add(message) } + fun setMessages(message: List>): Builder fun build(): MessageGroup } + //TODO: add override annotation + fun toBuilder(): Builder = builder().setMessages(messages) + class CollectionBuilder { private val elements: MutableList> = mutableListOf() diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt index 711d16446..777855d20 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt @@ -28,6 +28,33 @@ data class MessageId( /** The subsequence is not mutable by default */ val subsequence: List = emptyList(), ) { + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as MessageId + + if (sessionAlias != other.sessionAlias) return false + if (direction != other.direction) return false + if (sequence != other.sequence) return false + if (timestamp != other.timestamp) return false + return subsequence == other.subsequence + } + + override fun hashCode(): Int { + var result = sessionAlias.hashCode() + result = 31 * result + direction.hashCode() + result = 31 * result + sequence.hashCode() + result = 31 * result + timestamp.hashCode() + result = 31 * result + subsequence.hashCode() + return result + } + + override fun toString(): String { + return "MessageId(sessionAlias='$sessionAlias', direction=$direction, sequence=$sequence, timestamp=$timestamp, subsequence=$subsequence)" + } + @AutoBuilder interface Builder { val sessionAlias: String @@ -40,15 +67,15 @@ data class MessageId( fun setSequence(sequence: Long): Builder fun setTimestamp(timestamp: Instant): Builder fun subsequenceBuilder(): CollectionBuilder - fun addSubsequence(subsequnce: Int): Builder = apply { - subsequenceBuilder().add(subsequnce) + fun addSubsequence(subsequence: Int): Builder = apply { + subsequenceBuilder().add(subsequence) } fun setSubsequence(subsequence: List): Builder fun build(): MessageId } - fun toBuilder(): MessageId.Builder = AutoBuilder_MessageId_Builder(this) + fun toBuilder(): Builder = AutoBuilder_MessageId_Builder(this) companion object { @JvmStatic diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt index 4bb2d5274..e7e82c9ae 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt @@ -92,6 +92,21 @@ class ParsedMessage private constructor( override fun build(): ParsedMessage } + //TODO: add override annotation + fun toBuilder(): Builder> { + return when { + body === DEFAULT_BODY -> FromRawBuilderImpl(bodySupplier).setRawBody(rawBody) + bodySupplier === DEFAULT_BODY_SUPPLIER -> FromMapBuilderImpl().setBody(body) + else -> error("Message can't be convert to builder $this") + }.apply { + setId(id) + setMetadata(metadata) + setProtocol(protocol) + setType(type) + eventId?.let(::setEventId) + } + } + interface FromRawBuilder : Builder { fun setRawBody(rawBody: ByteBuf): FromRawBuilder } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt index 5bcd7b787..98de48fb0 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt @@ -18,6 +18,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.google.auto.value.AutoBuilder import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufUtil.hexDump import io.netty.buffer.Unpooled data class RawMessage( @@ -28,11 +29,40 @@ data class RawMessage( /** The body is not mutable by default */ override val body: ByteBuf = Unpooled.EMPTY_BUFFER, ) : Message { + + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as RawMessage + + if (id != other.id) return false + if (eventId != other.eventId) return false + if (metadata != other.metadata) return false + if (protocol != other.protocol) return false + return body == other.body + } + + override fun hashCode(): Int { + var result = id.hashCode() + result = 31 * result + (eventId?.hashCode() ?: 0) + result = 31 * result + metadata.hashCode() + result = 31 * result + protocol.hashCode() + result = 31 * result + body.hashCode() + return result + } + + override fun toString(): String { + return "RawMessage(id=$id, eventId=$eventId, metadata=$metadata, protocol='$protocol', body=${hexDump(body)})" + } + @AutoBuilder interface Builder : Message.Builder { - val body: ByteBuf + val body: ByteBuf fun setBody(body: ByteBuf): Builder + fun setBody(data: ByteArray): Builder = setBody(Unpooled.wrappedBuffer(data)) override fun addMetadataProperty(key: String, value: String): Builder = this.apply { @@ -42,7 +72,10 @@ data class RawMessage( override fun build(): RawMessage } + fun toBuilder(): Builder = AutoBuilder_RawMessage_Builder(this) + companion object { + val EMPTY = RawMessage() @JvmStatic From ca9e6d873ed96a42416e2f1f510920d958f909bc Mon Sep 17 00:00:00 2001 From: Oleg Smelov Date: Tue, 18 Jul 2023 17:51:08 +0400 Subject: [PATCH 137/154] "transport-group" value added to QueueAttribute enum --- .../exactpro/th2/common/schema/message/QueueAttribute.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/exactpro/th2/common/schema/message/QueueAttribute.java b/src/main/java/com/exactpro/th2/common/schema/message/QueueAttribute.java index 612211024..50407db01 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/QueueAttribute.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/QueueAttribute.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -12,6 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.exactpro.th2.common.schema.message; public enum QueueAttribute { @@ -22,7 +23,8 @@ public enum QueueAttribute { RAW("raw"), PARSED("parsed"), STORE("store"), - EVENT("event"); + EVENT("event"), + TRANSPORT_GROUP("transport-group"); private final String value; From da4cdabbb0c4919162186e9feb3cb5c3a761a969 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 19 Jul 2023 11:29:47 +0400 Subject: [PATCH 138/154] [TH2-4989] Corrected toBuilder interface for ParsedMessage --- .../message/impl/rabbitmq/transport/ParsedMessage.kt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt index e7e82c9ae..8b5d71b88 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt @@ -93,12 +93,9 @@ class ParsedMessage private constructor( } //TODO: add override annotation - fun toBuilder(): Builder> { - return when { - body === DEFAULT_BODY -> FromRawBuilderImpl(bodySupplier).setRawBody(rawBody) - bodySupplier === DEFAULT_BODY_SUPPLIER -> FromMapBuilderImpl().setBody(body) - else -> error("Message can't be convert to builder $this") - }.apply { + fun toBuilder(): FromMapBuilder { + return FromMapBuilderImpl().apply { + setBody(body) setId(id) setMetadata(metadata) setProtocol(protocol) From f6efed90324134cfa177884fd8a27e605d519520 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 19 Jul 2023 12:39:39 +0400 Subject: [PATCH 139/154] [TH2-4989] Added test for toBuilder method --- .../impl/rabbitmq/transport/ParsedMessage.kt | 15 +- .../rabbitmq/transport/ParsedMessageTest.kt | 169 ++++++++++++++++++ 2 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt index 8b5d71b88..57784b7a3 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt @@ -94,14 +94,13 @@ class ParsedMessage private constructor( //TODO: add override annotation fun toBuilder(): FromMapBuilder { - return FromMapBuilderImpl().apply { - setBody(body) - setId(id) - setMetadata(metadata) - setProtocol(protocol) - setType(type) - eventId?.let(::setEventId) - } + return FromMapBuilderImpl() + .setBody(body) + .setId(id) + .setMetadata(metadata) + .setProtocol(protocol) + .setType(type) + .also { eventId?.let(it::setEventId) } } interface FromRawBuilder : Builder { diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt new file mode 100644 index 000000000..324259f80 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt @@ -0,0 +1,169 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport + +import org.junit.jupiter.api.Test +import java.time.Instant +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class ParsedMessageTest { + + private val testBody = hashMapOf("test-field" to "test-filed-value") + private val testMetadata = hashMapOf("test-property" to "test-property-value") + private val testTimestamp = Instant.now() + + @Test + fun `builder test`() { + val builder = ParsedMessage.builder().apply { + setType(TEST_MESSAGE_TYPE) + setProtocol(TEST_PROTOCOL) + setBody(testBody) + setMetadata(testMetadata) + idBuilder().apply { + setSessionAlias(TEST_SESSION_ALIAS) + setDirection(Direction.OUTGOING) + setSequence(TEST_SEQUENCE) + setSubsequence(TEST_SUB_SEQUENCE) + setTimestamp(testTimestamp) + } + setEventId(EventId.builder().apply { + setBook(TEST_BOOK) + setScope(TEST_SCOPE) + setTimestamp(testTimestamp) + setId(TEST_EVENT_ID) + }.build()) + } + + with(builder) { + assertEquals(TEST_MESSAGE_TYPE, type) + assertEquals(TEST_PROTOCOL, protocol) +// assertEquals(body, builder.bodyBuilder) +// assertEquals(metadata, builder.metadataBuilder()) + with(idBuilder()) { + assertEquals(TEST_SESSION_ALIAS, sessionAlias) + assertEquals(Direction.OUTGOING, direction) + assertEquals(TEST_SEQUENCE, sequence) +// assertEquals(TEST_SUB_SEQUENCE, subsequenceBuilder()) + assertEquals(testTimestamp, timestamp) + } + with(assertNotNull(eventId)) { + assertEquals(TEST_BOOK, book) + assertEquals(TEST_SCOPE, scope) + assertEquals(testTimestamp, testTimestamp) + assertEquals(TEST_EVENT_ID, id) + } + } + + val mesasge = builder.build() + + with(mesasge) { + assertEquals(TEST_MESSAGE_TYPE, type) + assertEquals(TEST_PROTOCOL, protocol) + assertEquals(testBody, body) + assertEquals(testMetadata, metadata) + with(id) { + assertEquals(TEST_SESSION_ALIAS, sessionAlias) + assertEquals(Direction.OUTGOING, direction) + assertEquals(TEST_SEQUENCE, sequence) + assertEquals(TEST_SUB_SEQUENCE, subsequence) + assertEquals(testTimestamp, timestamp) + } + with(assertNotNull(eventId)) { + assertEquals(TEST_BOOK, book) + assertEquals(TEST_SCOPE, scope) + assertEquals(testTimestamp, testTimestamp) + assertEquals(TEST_EVENT_ID, id) + } + } + } + + @Test + fun `toBuilder test`() { + val message = ParsedMessage.builder().apply { + setType(TEST_MESSAGE_TYPE) + setProtocol(TEST_PROTOCOL) + setBody(testBody) + setMetadata(testMetadata) + idBuilder().apply { + setSessionAlias(TEST_SESSION_ALIAS) + setDirection(Direction.OUTGOING) + setSequence(TEST_SEQUENCE) + setSubsequence(TEST_SUB_SEQUENCE) + setTimestamp(testTimestamp) + } + setEventId(EventId.builder().apply { + setBook(TEST_BOOK) + setScope(TEST_SCOPE) + setTimestamp(testTimestamp) + setId(TEST_EVENT_ID) + }.build()) + }.build() + + val builder = message.toBuilder() + with(builder) { + assertEquals(TEST_MESSAGE_TYPE, type) + assertEquals(TEST_PROTOCOL, protocol) +// assertEquals(body, builder.bodyBuilder) +// assertEquals(metadata, builder.metadataBuilder()) + with(idBuilder()) { + assertEquals(TEST_SESSION_ALIAS, sessionAlias) + assertEquals(Direction.OUTGOING, direction) + assertEquals(TEST_SEQUENCE, sequence) +// assertEquals(TEST_SUB_SEQUENCE, subsequenceBuilder()) + assertEquals(testTimestamp, timestamp) + } + with(assertNotNull(eventId)) { + assertEquals(TEST_BOOK, book) + assertEquals(TEST_SCOPE, scope) + assertEquals(testTimestamp, testTimestamp) + assertEquals(TEST_EVENT_ID, id) + } + } + + with(builder.build()) { + assertEquals(TEST_MESSAGE_TYPE, type) + assertEquals(TEST_PROTOCOL, protocol) + assertEquals(testBody, body) + assertEquals(testMetadata, metadata) + with(id) { + assertEquals(TEST_SESSION_ALIAS, sessionAlias) + assertEquals(Direction.OUTGOING, direction) + assertEquals(TEST_SEQUENCE, sequence) + assertEquals(TEST_SUB_SEQUENCE, subsequence) + assertEquals(testTimestamp, timestamp) + } + with(assertNotNull(eventId)) { + assertEquals(TEST_BOOK, book) + assertEquals(TEST_SCOPE, scope) + assertEquals(testTimestamp, testTimestamp) + assertEquals(TEST_EVENT_ID, id) + } + } + } + + companion object { + private const val TEST_BOOK = "test-book" + private const val TEST_SCOPE = "test-scope" + private const val TEST_EVENT_ID = "test-event-id" + private const val TEST_PROTOCOL = "test-protocol" + private const val TEST_SESSION_ALIAS = "test-session-alias" + private const val TEST_MESSAGE_TYPE = "test-message-type" + private val TEST_SEQUENCE = Random.nextLong() + private val TEST_SUB_SEQUENCE = listOf(Random.nextInt(), Random.nextInt()) + } +} \ No newline at end of file From a9395ed78272b9f8115364bfe482624320048fac Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Wed, 19 Jul 2023 13:13:46 +0400 Subject: [PATCH 140/154] [TH2-4989] Added getters to Map/Collection builders --- .../transport/builders/CollectionBuilder.kt | 4 +- .../rabbitmq/transport/builders/MapBuilder.kt | 7 +++ .../rabbitmq/transport/ParsedMessageTest.kt | 47 +++++++++++++++---- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/CollectionBuilder.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/CollectionBuilder.kt index d55b47502..965feebcc 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/CollectionBuilder.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/CollectionBuilder.kt @@ -23,7 +23,9 @@ class CollectionBuilder { get() = elements.size fun isEmpty(): Boolean = elements.isEmpty() - + + operator fun get(index: Int): T = elements[index] + fun add(el: T): CollectionBuilder = apply { elements += el } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/MapBuilder.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/MapBuilder.kt index d85f6c43d..e30e5c328 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/MapBuilder.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/builders/MapBuilder.kt @@ -19,12 +19,19 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders class MapBuilder( private val innerMap: MutableMap = hashMapOf(), ) { + val size: Int + get() = innerMap.size + + fun contains(key: K): Boolean = innerMap.contains(key) + operator fun get(key: K): V? = innerMap[key] fun put(key: K, value: V): MapBuilder = apply { innerMap[key] = value } + fun putAll(from: Map): MapBuilder = apply { innerMap.putAll(from) } + fun build(): Map { return innerMap } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt index 324259f80..2560cb198 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt @@ -16,6 +16,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertAll import java.time.Instant import kotlin.random.Random import kotlin.test.assertEquals @@ -52,13 +53,28 @@ class ParsedMessageTest { with(builder) { assertEquals(TEST_MESSAGE_TYPE, type) assertEquals(TEST_PROTOCOL, protocol) -// assertEquals(body, builder.bodyBuilder) -// assertEquals(metadata, builder.metadataBuilder()) + with(bodyBuilder()) { + assertEquals(testBody.size, size) + assertAll(testBody.map { (key, value) -> + { assertEquals(value, get(key), "Check '$key' field") } + }) + } + with(metadataBuilder()) { + assertEquals(testMetadata.size, size) + assertAll(testMetadata.map { (key, value) -> + { assertEquals(value, get(key), "Check '$key' field") } + }) + } with(idBuilder()) { assertEquals(TEST_SESSION_ALIAS, sessionAlias) assertEquals(Direction.OUTGOING, direction) assertEquals(TEST_SEQUENCE, sequence) -// assertEquals(TEST_SUB_SEQUENCE, subsequenceBuilder()) + with(subsequenceBuilder()) { + assertEquals(TEST_SUB_SEQUENCE.size, size) + assertAll(TEST_SUB_SEQUENCE.mapIndexed { index, value -> + { assertEquals(value, get(index), "Check '$index' index") } + }) + } assertEquals(testTimestamp, timestamp) } with(assertNotNull(eventId)) { @@ -69,9 +85,7 @@ class ParsedMessageTest { } } - val mesasge = builder.build() - - with(mesasge) { + with(builder.build()) { assertEquals(TEST_MESSAGE_TYPE, type) assertEquals(TEST_PROTOCOL, protocol) assertEquals(testBody, body) @@ -118,13 +132,28 @@ class ParsedMessageTest { with(builder) { assertEquals(TEST_MESSAGE_TYPE, type) assertEquals(TEST_PROTOCOL, protocol) -// assertEquals(body, builder.bodyBuilder) -// assertEquals(metadata, builder.metadataBuilder()) + with(bodyBuilder()) { + assertEquals(testBody.size, size) + assertAll(testBody.map { (key, value) -> + { assertEquals(value, get(key), "Check '$key' field") } + }) + } + with(metadataBuilder()) { + assertEquals(testMetadata.size, size) + assertAll(testMetadata.map { (key, value) -> + { assertEquals(value, get(key), "Check '$key' field") } + }) + } with(idBuilder()) { assertEquals(TEST_SESSION_ALIAS, sessionAlias) assertEquals(Direction.OUTGOING, direction) assertEquals(TEST_SEQUENCE, sequence) -// assertEquals(TEST_SUB_SEQUENCE, subsequenceBuilder()) + with(subsequenceBuilder()) { + assertEquals(TEST_SUB_SEQUENCE.size, size) + assertAll(TEST_SUB_SEQUENCE.mapIndexed { index, value -> + { assertEquals(value, get(index), "Check '$index' index") } + }) + } assertEquals(testTimestamp, timestamp) } with(assertNotNull(eventId)) { From 69a096c6dc63fdfa06fc030f7dc90a12f7fc298c Mon Sep 17 00:00:00 2001 From: Oleg Smelov <45400511+lumber1000@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:11:07 +0400 Subject: [PATCH 141/154] [TH2-3673] Filters behaviour (#270) Pin filters behaviour changed: conditions inside the message and metadata now combined as "and" --- README.md | 1 + .../strategy/impl/AbstractFilterStrategy.java | 4 +- .../impl/AbstractTh2MsgFilterStrategy.java | 9 +- .../strategy/impl/AnyMessageFilterStrategy.kt | 4 + .../impl/TestAnyMessageFilterStrategy.kt | 479 +++++++++++++++++- 5 files changed, 463 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 599df2e36..6d8424f5a 100644 --- a/README.md +++ b/README.md @@ -467,6 +467,7 @@ dependencies { ### 5.3.0-dev + Implemented message routers used th2 transport protocol for interaction ++ Pin filters behaviour changed: conditions inside the message and metadata now combined as "and" #### Updated: + cradle: `5.1.1-dev` diff --git a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java index fee14236e..7523d6208 100644 --- a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java +++ b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -52,7 +52,7 @@ public boolean verify(T message, List routerFilters) { protected abstract Map getFields(T message); private boolean checkValues(Map messageFields, MultiValuedMap fieldFilters) { - return fieldFilters.isEmpty() || fieldFilters.keys().stream().anyMatch(fieldName -> { + return fieldFilters.isEmpty() || fieldFilters.keys().stream().allMatch(fieldName -> { String messageValue = messageFields.get(fieldName); Collection filters = fieldFilters.get(fieldName); return !filters.isEmpty() && filters.stream().allMatch(filter -> FieldValueChecker.checkFieldValue(filter, messageValue)); diff --git a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java index 01a9f8302..1fb4a7578 100644 --- a/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java +++ b/src/main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -15,7 +15,6 @@ package com.exactpro.th2.common.schema.filter.strategy.impl; - import com.exactpro.th2.common.grpc.Message; import com.exactpro.th2.common.grpc.MessageID; import com.exactpro.th2.common.grpc.MessageMetadata; @@ -23,7 +22,6 @@ import java.util.Map; import java.util.stream.Collectors; - public abstract class AbstractTh2MsgFilterStrategy extends AbstractFilterStrategy { public static final String BOOK_KEY = "book"; @@ -31,6 +29,7 @@ public abstract class AbstractTh2MsgFilterStrategy extends AbstractFilterStrateg public static final String SESSION_ALIAS_KEY = "session_alias"; public static final String MESSAGE_TYPE_KEY = "message_type"; public static final String DIRECTION_KEY = "direction"; + public static final String PROTOCOL_KEY = "protocol"; @Override public Map getFields(com.google.protobuf.Message message) { @@ -50,7 +49,8 @@ public Map getFields(com.google.protobuf.Message message) { SESSION_GROUP_KEY, sessionGroup.isEmpty() ? sessionAlias : sessionGroup, SESSION_ALIAS_KEY, sessionAlias, MESSAGE_TYPE_KEY, metadata.getMessageType(), - DIRECTION_KEY, messageID.getDirection().name() + DIRECTION_KEY, messageID.getDirection().name(), + PROTOCOL_KEY, metadata.getProtocol() ); messageFields.putAll(metadataMsgFields); @@ -59,5 +59,4 @@ public Map getFields(com.google.protobuf.Message message) { } public abstract Message parseMessage(com.google.protobuf.Message message); - } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt index 5d76c600e..44abeac2f 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt @@ -34,20 +34,24 @@ object AnyMessageFilterStrategy : AbstractFilterStrategy() { val metadata = message.message.metadata val sessionAlias = metadata.id.connectionId.sessionAlias val sessionGroup = metadata.id.connectionId.sessionGroup + result.putAll(metadata.propertiesMap) result[AbstractTh2MsgFilterStrategy.BOOK_KEY] = metadata.id.bookName result[AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY] = sessionGroup.ifEmpty { sessionAlias } result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = sessionAlias result[AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY] = metadata.messageType result[AbstractTh2MsgFilterStrategy.DIRECTION_KEY] = metadata.id.direction.name + result[AbstractTh2MsgFilterStrategy.PROTOCOL_KEY] = metadata.protocol } message.hasRawMessage() -> { val metadata = message.rawMessage.metadata val sessionAlias = metadata.id.connectionId.sessionAlias val sessionGroup = metadata.id.connectionId.sessionGroup + result.putAll(metadata.propertiesMap) result[AbstractTh2MsgFilterStrategy.BOOK_KEY] = metadata.id.bookName result[AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY] = sessionGroup.ifEmpty { sessionAlias } result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = metadata.id.connectionId.sessionAlias result[AbstractTh2MsgFilterStrategy.DIRECTION_KEY] = metadata.id.direction.name + result[AbstractTh2MsgFilterStrategy.PROTOCOL_KEY] = metadata.protocol } else -> throw IllegalStateException("Message has not messages: ${message.toJson()}") } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt index bd59f34bc..b1a60e737 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/TestAnyMessageFilterStrategy.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import com.exactpro.th2.common.event.bean.BaseTest.BOOK_NAME import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.Direction import com.exactpro.th2.common.grpc.RawMessage +import com.exactpro.th2.common.message.addField import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.toJson import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration @@ -33,6 +34,158 @@ import org.junit.jupiter.params.provider.Arguments.arguments import org.junit.jupiter.params.provider.MethodSource class TestAnyMessageFilterStrategy { + + @ParameterizedTest + @MethodSource("multipleFiltersMatch") + fun `matches any filter`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + anyMessage, + listOf( + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("message_type", FieldFilterConfiguration( + fieldName = "message_type", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test" + )) + } + ), + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("direction", FieldFilterConfiguration( + fieldName = "direction", + operation = FieldFilterOperation.EQUAL, + expectedValue = "FIRST" + )) + } + ) + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("rawMessagesBothFilters") + fun `matches with multiple metadata filters`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + anyMessage, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("session_alias", FieldFilterConfiguration( + fieldName = "session_alias", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-alias" + )) + put("direction", FieldFilterConfiguration( + fieldName = "direction", + operation = FieldFilterOperation.EQUAL, + expectedValue = "FIRST" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("parsedMessagesBothFilters") + fun `matches with multiple message filters`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + anyMessage, + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put("test-field1", FieldFilterConfiguration( + fieldName = "test-field1", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value1" + )) + put("test-field2", FieldFilterConfiguration( + fieldName = "test-field2", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithProperties") + fun `matches with multiple properties filters`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + anyMessage, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("prop-field1", FieldFilterConfiguration( + fieldName = "prop-field1", + operation = FieldFilterOperation.EQUAL, + expectedValue = "prop-value1" + )) + put("prop-field2", FieldFilterConfiguration( + fieldName = "prop-field2", + operation = FieldFilterOperation.EQUAL, + expectedValue = "prop-value2" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithMessageAndMetadataFilters") + fun `matches with multiple message and metadata filters`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + anyMessage, + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put("test-field1", FieldFilterConfiguration( + fieldName = "test-field1", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value1" + )) + put("test-field2", FieldFilterConfiguration( + fieldName = "test-field2", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + )) + }, + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("message_type", FieldFilterConfiguration( + fieldName = "message_type", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test" + )) + put("direction", FieldFilterConfiguration( + fieldName = "direction", + operation = FieldFilterOperation.EQUAL, + expectedValue = "FIRST" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messageWithProtocol") + fun `matches protocol metadata filter`(anyMessage: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + anyMessage, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put("protocol", FieldFilterConfiguration( + fieldName = "protocol", + operation = FieldFilterOperation.EQUAL, + expectedValue = "HTTP" + )) + } + ) + ) + assertEquals(expectMatch, match) { "The message ${anyMessage.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + @ParameterizedTest @MethodSource("parsedMessages") fun `matches the parsed message by message type with single filter`(anyMessage: AnyMessage, expectMatch: Boolean) { @@ -82,48 +235,320 @@ class TestAnyMessageFilterStrategy { expectedValue = "test-alias" )) } - )) + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithSameFilterFields") + fun `miss matches with the same filter fields`(message: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + message, + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value1" + ) + ) + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + ) + ) + } + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithMultipleFiltersWithSameFilterField") + fun `matches with multiple filters with the same filter fields`(message: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + message, + listOf( + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value1" + ) + ) + } + ), + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + ) + ) + } + ) + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithMultipleSameFields") + fun `matches message with multiple fields with same name`(message: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + message, + MqRouterFilterConfiguration( + message = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.NOT_EQUAL, + expectedValue = "test-value1" + ) + ) + put( + "test-field", FieldFilterConfiguration( + fieldName = "test-field", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test-value2" + ) + ) + } + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithOneProperty") + fun `matches message with properties`(message: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + message, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-property", FieldFilterConfiguration( + fieldName = "test-property", + operation = FieldFilterOperation.EQUAL, + expectedValue = "property-value" + ) + ) + } + ) + ) + + assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } + } + + @ParameterizedTest + @MethodSource("messagesWithPropertiesAndMetadata") + fun `matches message with properties and metadata`(message: AnyMessage, expectMatch: Boolean) { + val match = AnyMessageFilterStrategy.verify( + message, + MqRouterFilterConfiguration( + metadata = MultiMapUtils.newListValuedHashMap().apply { + put( + "test-property", FieldFilterConfiguration( + fieldName = "test-property", + operation = FieldFilterOperation.EQUAL, + expectedValue = "property-value" + ) + ) + + put("message_type", FieldFilterConfiguration( + fieldName = "message_type", + operation = FieldFilterOperation.EQUAL, + expectedValue = "test" + )) + } + ) + ) assertEquals(expectMatch, match) { "The message ${message.toJson()} was${if (expectMatch) "" else " not"} matched" } } companion object { - private val PARSED_MESSAGE_MATCH = AnyMessage.newBuilder().setMessage( - message(BOOK_NAME, "test", Direction.FIRST, "test-alias") - ).build() + private fun simpleMessageBuilder(messageType: String, direction: Direction, sessionAlias: String): AnyMessage { + return AnyMessage.newBuilder().setMessage( + message(BOOK_NAME, messageType, direction, sessionAlias) + ).build() + } + + private fun simpleRawMessageBuilder(sessionAlias: String, directionValue: Direction): AnyMessage { + return AnyMessage.newBuilder().setRawMessage( + RawMessage.newBuilder().apply { + metadataBuilder.idBuilder.apply { + connectionIdBuilder.sessionAlias = sessionAlias + direction = directionValue + } + } + ).build() + } - private val RAW_MESSAGE_MATCH = AnyMessage.newBuilder().setRawMessage( - RawMessage.newBuilder().apply { - metadataBuilder.idBuilder.apply { - connectionIdBuilder.sessionAlias = "test-alias" - direction = Direction.FIRST + private fun messageWithFieldsBuilder(messageType: String, direction: Direction, fields: List>): AnyMessage { + return AnyMessage.newBuilder().setMessage( + message(BOOK_NAME, messageType, direction, "test-alias").apply { + fields.forEach { addField(it.first, it.second) } } - } - ).build() + ).build() + } - private val PARSED_MESSAGE_MISS_MATCH = AnyMessage.newBuilder().setMessage( - message(BOOK_NAME, "test1", Direction.SECOND, "test-alias1") - ).build() + private fun messageWithPropertiesBuilder(messageType: String, direction: Direction, properties: List>): AnyMessage { + return AnyMessage.newBuilder().setMessage( + message(BOOK_NAME, messageType, direction, "test-alias").apply { + properties.forEach { metadataBuilder.putProperties(it.first, it.second) } + } + ).build() + } + + private fun rawMessageWithOnePropertyBuilder(propertyKey: String, propertyValue: String): AnyMessage { + return AnyMessage.newBuilder().setRawMessage( + RawMessage.newBuilder().apply { + metadataBuilder.putProperties(propertyKey, propertyValue) + } + ).build() + } - private val RAW_MESSAGE_MISS_MATCH = AnyMessage.newBuilder().setRawMessage( - RawMessage.newBuilder().apply { - metadataBuilder.idBuilder.apply { - connectionIdBuilder.sessionAlias = "test-alias1" - direction = Direction.SECOND + private fun messageWithOnePropertyBuilder(messageType: String, propertyKey: String, propertyValue: String): AnyMessage { + return messageWithPropertiesBuilder(messageType, Direction.FIRST, listOf(Pair(propertyKey, propertyValue))) + } + + private fun messageWithProtocolBuilder(protocol: String): AnyMessage { + return AnyMessage.newBuilder().setMessage( + message(BOOK_NAME, "test", Direction.FIRST, "test-alias").apply { + metadataBuilder.protocol = protocol } - } - ).build() + ).build() + } @JvmStatic - fun parsedMessages(): List = listOf( - arguments(PARSED_MESSAGE_MATCH, true), - arguments(PARSED_MESSAGE_MISS_MATCH, false) + fun messageWithProtocol(): List = listOf( + arguments(messageWithProtocolBuilder("HTTP"), true), + arguments(messageWithProtocolBuilder("FTP"), false), ) @JvmStatic fun messages(): List = listOf( - arguments(RAW_MESSAGE_MATCH, true), - arguments(RAW_MESSAGE_MISS_MATCH, false) + arguments(simpleRawMessageBuilder("test-alias", Direction.FIRST), true), + arguments(simpleRawMessageBuilder("test-alias1", Direction.SECOND), false) ) + parsedMessages() + + @JvmStatic + fun parsedMessages(): List = listOf( + arguments(simpleMessageBuilder("test", Direction.FIRST, "test-alias"), true), + arguments(simpleMessageBuilder("test1", Direction.SECOND, "test-alias1"), false) + ) + + @JvmStatic + fun messagesWithProperties(): List = listOf( + arguments(messageWithPropertiesBuilder("test", Direction.FIRST, listOf( + Pair("prop-field1", "prop-value1"), Pair("prop-field2", "prop-value2"))), true), + arguments(messageWithPropertiesBuilder("test", Direction.FIRST, listOf( + Pair("prop-field1", "prop-value-wrong"), Pair("prop-field2", "prop-value2"))), false), + arguments(messageWithPropertiesBuilder("test", Direction.FIRST, listOf( + Pair("prop-field1", "prop-value1"), Pair("prop-field2", "prop-value-wrong"))), false), + arguments(messageWithPropertiesBuilder("test", Direction.FIRST, listOf( + Pair("prop-field1", "prop-value-wrong"), Pair("prop-field2", "prop-value-wrong"))), false) + ) + + @JvmStatic + fun messagesWithMultipleSameFields(): List = listOf( + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field", "test-value1"), Pair("test-field", "test-value2"))), true) + ) + + @JvmStatic + fun messagesWithSameFilterFields(): List = listOf( + arguments(messageWithFieldsBuilder("test", Direction.FIRST, listOf(Pair("test-field", "test-value1"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, listOf(Pair("test-field", "test-value2"))), false) + ) + + @JvmStatic + fun messagesWithMultipleFiltersWithSameFilterField(): List = listOf( + arguments(messageWithFieldsBuilder("test", Direction.FIRST, listOf(Pair("test-field", "test-value1"))), true), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, listOf(Pair("test-field", "test-value2"))), true) + ) + + @JvmStatic + fun multipleFiltersMatch(): List = listOf( + arguments(simpleMessageBuilder("test", Direction.FIRST, "test-alias"), true), + arguments(simpleMessageBuilder("test", Direction.SECOND, "test-alias"), true), + arguments(simpleMessageBuilder("test-wrong", Direction.FIRST, "test-alias"), true), + arguments(simpleMessageBuilder("test-wrong", Direction.SECOND, "test-alias"), false) + ) + + @JvmStatic + fun messagesWithMessageAndMetadataFilters() : List = listOf( + // fields full match + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value2"))), true), + + // metadata mismatch + arguments(messageWithFieldsBuilder("test", Direction.SECOND, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value2"))), false), + arguments(messageWithFieldsBuilder("test-wrong", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value2"))), false), + + // fields mismatch + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value2"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value-wrong"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value-wrong"))), false), + + // one field and one metadata mismatch + arguments(messageWithFieldsBuilder("test", Direction.SECOND, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value2"))), false), + arguments(messageWithFieldsBuilder("test-wrong", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value-wrong"))), false) + ) + + @JvmStatic + fun parsedMessagesBothFilters() : List = listOf( + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value2"))), true), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value2"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value1"), Pair("test-field2", "test-value-wrong"))), false), + arguments(messageWithFieldsBuilder("test", Direction.FIRST, + listOf(Pair("test-field1", "test-value-wrong"), Pair("test-field2", "test-value-wrong"))), false) + ) + + @JvmStatic + fun messagesWithOneProperty() : List = listOf( + arguments(messageWithOnePropertyBuilder("test", "test-property", "property-value"), true), + arguments(messageWithOnePropertyBuilder("test", "test-property", "property-value-wrong"), false), + arguments(rawMessageWithOnePropertyBuilder("test-property", "property-value"), true), + arguments(rawMessageWithOnePropertyBuilder("test-property", "property-value-wrong"), false) + ) + + @JvmStatic + fun messagesWithPropertiesAndMetadata() : List = listOf( + arguments(messageWithOnePropertyBuilder("test", "test-property", "property-value"), true), + arguments(messageWithOnePropertyBuilder("test", "test-property", "property-value-wrong"), false), + arguments(messageWithOnePropertyBuilder("test-wrong", "test-property", "property-value"), false) + ) + + @JvmStatic + fun rawMessagesBothFilters() : List = listOf( + arguments(simpleRawMessageBuilder("test-alias", Direction.FIRST), true), + arguments(simpleRawMessageBuilder("test-alias", Direction.SECOND), false), + arguments(simpleRawMessageBuilder("test-alias-wrong-value", Direction.FIRST), false), + arguments(simpleRawMessageBuilder("test-alias-wrong-value", Direction.SECOND), false) + ) } } \ No newline at end of file From 5a290063b824228270ef5892349c158845526324 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Mon, 31 Jul 2023 12:51:34 +0400 Subject: [PATCH 142/154] [TH2-5004] added filter by protocol for transport --- .../impl/rabbitmq/transport/TransportUtils.kt | 4 + .../rabbitmq/transport/TransportUtilsTest.kt | 90 ++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt index 1881d62c6..9dec92e44 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtils.kt @@ -21,6 +21,7 @@ import com.exactpro.th2.common.grpc.Direction.SECOND import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.BOOK_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.DIRECTION_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.PROTOCOL_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.checkFieldValue @@ -54,6 +55,9 @@ fun Collection.filter(batch: GroupBatch): GroupBatch? { if (!filterSet.metadata[DIRECTION_KEY].verify(batch.groups) { id.direction.proto.name }) { return@forEach } + if (!filterSet.metadata[PROTOCOL_KEY].verify(batch.groups) { protocol }) { + return@forEach + } return batch } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt index 40b4c6819..619c8dcf8 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportUtilsTest.kt @@ -19,6 +19,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.BOOK_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.DIRECTION_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY +import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.PROTOCOL_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY import com.exactpro.th2.common.schema.filter.strategy.impl.AbstractTh2MsgFilterStrategy.SESSION_GROUP_KEY import com.exactpro.th2.common.schema.message.configuration.FieldFilterConfiguration @@ -50,6 +51,8 @@ class TransportUtilsTest { private val directionA = "SECOND" private val directionB = "FIRST" + private val protocolA = "protocolA" + private val protocolB = "protocolB" private val routerFilters = listOf( MqRouterFilterConfiguration( @@ -66,6 +69,7 @@ class TransportUtilsTest { put(SESSION_ALIAS_KEY, FieldFilterConfiguration(SESSION_ALIAS_KEY, null, NOT_EMPTY)) put(MESSAGE_TYPE_KEY, FieldFilterConfiguration(MESSAGE_TYPE_KEY, null, NOT_EMPTY)) put(DIRECTION_KEY, FieldFilterConfiguration(DIRECTION_KEY, directionB, NOT_EQUAL)) + put(PROTOCOL_KEY, FieldFilterConfiguration(PROTOCOL_KEY, protocolA, EQUAL)) }, emptyMultiMap() ), @@ -83,6 +87,7 @@ class TransportUtilsTest { put(SESSION_ALIAS_KEY, FieldFilterConfiguration(SESSION_ALIAS_KEY, null, NOT_EMPTY)) put(MESSAGE_TYPE_KEY, FieldFilterConfiguration(MESSAGE_TYPE_KEY, null, NOT_EMPTY)) put(DIRECTION_KEY, FieldFilterConfiguration(DIRECTION_KEY, directionA, NOT_EQUAL)) + put(PROTOCOL_KEY, FieldFilterConfiguration(PROTOCOL_KEY, protocolB, EQUAL)) }, emptyMultiMap() ) @@ -106,21 +111,97 @@ class TransportUtilsTest { val batch = GroupBatch.builder() .setBook("") .setSessionGroup("") - .build() + .addGroup( + MessageGroup.builder() + .addMessage( + ParsedMessage.builder() + .setId( + MessageId.builder() + .setSessionAlias("") + .setDirection(Direction.OUTGOING) + .setSequence(1) + .setTimestamp(Instant.now()) + .build() + ) + .setType("") + .setBody(emptyMap()) + .build() + ) + .build() + ).build() assertNull(routerFilters.filter(batch)) }, DynamicTest.dynamicTest("only book match") { val batch = GroupBatch.builder() .setBook(bookA) .setSessionGroup("") - .build() + .addGroup( + MessageGroup.builder() + .addMessage( + ParsedMessage.builder() + .setId( + MessageId.builder() + .setSessionAlias("") + .setDirection(Direction.OUTGOING) + .setSequence(1) + .setTimestamp(Instant.now()) + .build() + ) + .setType("") + .setBody(emptyMap()) + .build() + ) + .build() + ).build() assertNull(routerFilters.filter(batch)) }, DynamicTest.dynamicTest("only book and group match") { val batch = GroupBatch.builder() .setBook(bookA) .setSessionGroup(groupA) - .build() + .addGroup( + MessageGroup.builder() + .addMessage( + ParsedMessage.builder() + .setId( + MessageId.builder() + .setSessionAlias("") + .setDirection(Direction.OUTGOING) + .setSequence(1) + .setTimestamp(Instant.now()) + .build() + ) + .setType("") + .setBody(emptyMap()) + .build() + ) + .build() + ).build() + assertNull(routerFilters.filter(batch)) + }, + DynamicTest.dynamicTest("only book, group, protocol match") { + val batch = GroupBatch.builder() + .setBook(bookA) + .setSessionGroup(groupA) + .addGroup( + MessageGroup.builder() + .addMessage( + ParsedMessage.builder() + .setProtocol(protocolA) + .setId( + MessageId.builder() + .setSessionAlias("") + .setDirection(Direction.OUTGOING) + .setSequence(1) + .setTimestamp(Instant.now()) + .build() + ) + .setType("") + .setBody(emptyMap()) + .build() + ) + .build() + ).build() assertNull(routerFilters.filter(batch)) }, DynamicTest.dynamicTest("with partial message match") { @@ -131,6 +212,7 @@ class TransportUtilsTest { MessageGroup.builder() .addMessage( ParsedMessage.builder() + .setProtocol(protocolA) .setId( MessageId.builder() .setSessionAlias("") @@ -157,6 +239,7 @@ class TransportUtilsTest { ParsedMessage.builder() .setType(msgType) .setBody(emptyMap()) + .setProtocol(protocolA) .apply { idBuilder() .setSessionAlias("alias") @@ -179,6 +262,7 @@ class TransportUtilsTest { ParsedMessage.builder() .setType(msgType) .setBody(emptyMap()) + .setProtocol(protocolB) .apply { idBuilder() .setSessionAlias("alias") From dad1031b5ca0203136117d78839e1c45f966ee7d Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Sat, 12 Aug 2023 12:56:35 +0400 Subject: [PATCH 143/154] [TH2-4887] Increment maintenance version 5.3.2 * Updated README --- README.md | 13 +++++++++++-- gradle.properties | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6d8424f5a..c1a5286f0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (5.3.1) +# th2 common library (Java) (5.3.2) ## Usage @@ -455,6 +455,16 @@ dependencies { ## Release notes +### 5.3.2-dev + +#### Fix ++ Pin filters behaviour changed: conditions inside the message and metadata now combined as "and" + +#### Feature ++ Added `protocol` field name for MQ pin filter ++ Added `hashCode`, `equals`, `toString`, `toBuilder` for th2 transport classes ++ Added `get` method for th2 transport `MapBuilder`, `CollectionBuilder` classes + ### 5.3.1-dev + Auto-print git metadata from `git.properties` resource file. @@ -467,7 +477,6 @@ dependencies { ### 5.3.0-dev + Implemented message routers used th2 transport protocol for interaction -+ Pin filters behaviour changed: conditions inside the message and metadata now combined as "and" #### Updated: + cradle: `5.1.1-dev` diff --git a/gradle.properties b/gradle.properties index 4b6b65a46..f3fc81b75 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.3.1 +release_version=5.3.2 description='th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j kapt.include.compile.classpath=false From 8e6d08eba3f2cbb0decc5398176d07f848276a59 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 15 Aug 2023 13:47:08 +0400 Subject: [PATCH 144/154] [TH2-5035] Updated kubernetes-client: `6.1.1` to `6.8.0` * okhttp: `4.10.0` to `4.11.0` * okio: `3.0.0` to `3.5.0` * bom: `4.4.0-dev` to `4.5.0-dev` --- README.md | 9 ++++++++- build.gradle | 14 +++++++++----- gradle.properties | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c1a5286f0..c5721c289 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (5.3.2) +# th2 common library (Java) (5.4.0) ## Usage @@ -455,6 +455,13 @@ dependencies { ## Release notes +### 5.4.0-dev +#### Updated ++ bom: `4.4.0-dev` to `4.5.0-dev` ++ kubernetes-client: `6.1.1` to `6.8.0` + + okhttp: `4.10.0` to `4.11.0` + + okio: `3.0.0` to `3.5.0` + ### 5.3.2-dev #### Fix diff --git a/build.gradle b/build.gradle index f7bf74f55..b37f806d3 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,6 @@ import com.github.jk1.license.filter.LicenseBundleNormalizer import com.github.jk1.license.render.JsonReportRenderer +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile buildscript { repositories { @@ -7,7 +8,7 @@ buildscript { } ext { - kotlin_version = "1.6.21" + kotlin_version = "1.8.22" } dependencies { @@ -173,7 +174,7 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:4.4.0") + api platform("com.exactpro.th2:bom:4.5.0-TH2-5035-+") api('com.exactpro.th2:grpc-common:4.3.0-dev') { because('protobuf transport is main now, this dependnecy should be moved to grpc, mq protobuf modules after splitting') } @@ -240,10 +241,13 @@ dependencies { implementation 'io.prometheus:simpleclient_httpserver' implementation 'io.prometheus:simpleclient_log4j2' - implementation('com.squareup.okhttp3:okhttp:4.10.0') { + implementation('com.squareup.okio:okio:3.4.0') { + because('fix vulnerability in transitive dependency ') + } + implementation('com.squareup.okhttp3:okhttp:4.11.0') { because('fix vulnerability in transitive dependency ') } - implementation('io.fabric8:kubernetes-client:6.1.1') { + implementation('io.fabric8:kubernetes-client:6.8.0') { exclude group: 'com.fasterxml.jackson.dataformat', module: 'jackson-dataformat-yaml' } @@ -277,7 +281,7 @@ sourceSets { main.kotlin.srcDirs += "src/main/kotlin" } -tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach { +tasks.withType(KotlinCompile).configureEach { kotlinOptions.jvmTarget = "11" kotlinOptions.freeCompilerArgs += "-Xjvm-default=all" } diff --git a/gradle.properties b/gradle.properties index f3fc81b75..ff7e89a95 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.3.2 +release_version=5.4.0 description='th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j kapt.include.compile.classpath=false From c7a9376350e64c73aa9508d9713e8866e22ef24c Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 15 Aug 2023 19:25:14 +0400 Subject: [PATCH 145/154] [TH2-5035] Updated Readme * kotlin: `1.8.22` --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c5721c289..48f27c4f1 100644 --- a/README.md +++ b/README.md @@ -458,6 +458,7 @@ dependencies { ### 5.4.0-dev #### Updated + bom: `4.4.0-dev` to `4.5.0-dev` ++ kotlin: `1.8.22` + kubernetes-client: `6.1.1` to `6.8.0` + okhttp: `4.10.0` to `4.11.0` + okio: `3.0.0` to `3.5.0` From 0056a6245aca56de33885a67241ef9b1e42fd37b Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 15 Aug 2023 20:48:07 +0400 Subject: [PATCH 146/154] [TH2-5035] Updated suppressions.xml --- suppressions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suppressions.xml b/suppressions.xml index 50f2320cb..de58f6de3 100644 --- a/suppressions.xml +++ b/suppressions.xml @@ -1,7 +1,7 @@ - + ^pkg:maven/com\.exactpro\.th2/task-utils@.*$ cpe:/a:utils_project:utils From efe27e6afcdc1f3f1996d07b1c91d6e09763fe06 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 15 Aug 2023 20:59:15 +0400 Subject: [PATCH 147/154] [TH2-5035] corrected okio version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b37f806d3..b43a00141 100644 --- a/build.gradle +++ b/build.gradle @@ -241,7 +241,7 @@ dependencies { implementation 'io.prometheus:simpleclient_httpserver' implementation 'io.prometheus:simpleclient_log4j2' - implementation('com.squareup.okio:okio:3.4.0') { + implementation('com.squareup.okio:okio:3.5.0') { because('fix vulnerability in transitive dependency ') } implementation('com.squareup.okhttp3:okhttp:4.11.0') { From 8baca1def4fca96be1238b028ddd9923393c2b29 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 15 Aug 2023 21:35:58 +0400 Subject: [PATCH 148/154] [TH2-5035] use release version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b43a00141..78630087b 100644 --- a/build.gradle +++ b/build.gradle @@ -174,7 +174,7 @@ tasks.register('integrationTest', Test) { } dependencies { - api platform("com.exactpro.th2:bom:4.5.0-TH2-5035-+") + api platform("com.exactpro.th2:bom:4.5.0") api('com.exactpro.th2:grpc-common:4.3.0-dev') { because('protobuf transport is main now, this dependnecy should be moved to grpc, mq protobuf modules after splitting') } From c72989727efb25e3e3cc6e85d0e1bbd3b0d79fc4 Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Wed, 6 Sep 2023 12:12:05 +0400 Subject: [PATCH 149/154] [TH2-5007] MessageSubscriber supports single listener (#275) * The unsubscribe method of SubscribtionMonitor interface cancels RabbitMQ subscribtion --- README.md | 45 ++- gradle.properties | 2 +- .../common/schema/event/EventBatchRouter.java | 12 +- .../schema/event/EventBatchSubscriber.java | 9 +- .../grpc/router/AbstractGrpcRouter.java | 7 - .../common/schema/grpc/router/GrpcRouter.java | 7 +- .../common/schema/message/MessageRouter.java | 19 +- .../schema/message/MessageSubscriber.java | 28 +- .../rabbitmq/AbstractRabbitSubscriber.java | 122 ++------ .../connection/ConnectionManager.java | 2 +- .../common/schema/message/DeliveryMetadata.kt | 2 +- .../impl/rabbitmq/AbstractRabbitRouter.kt | 56 ++-- .../rabbitmq/custom/RabbitCustomRouter.kt | 17 +- .../group/RabbitMessageGroupBatchRouter.kt | 12 +- .../RabbitMessageGroupBatchSubscriber.kt | 6 +- .../NotificationEventBatchSubscriber.kt | 34 +-- .../transport/TransportGroupBatchRouter.kt | 10 +- .../TransportGroupBatchSubscriber.kt | 6 +- .../TestConfirmationMessageListenerWrapper.kt | 2 +- .../AbstractRabbitRouterIntegrationTest.kt | 226 +++++++++++++++ .../impl/rabbitmq/AbstractRabbitRouterTest.kt | 264 ++++++++++++++++++ .../impl/rabbitmq/TestMessageConverter.kt | 30 ++ .../group/TestRabbitMessageBatchRouter.kt | 40 +-- .../TransportGroupBatchRouterTest.kt | 41 +-- 24 files changed, 725 insertions(+), 274 deletions(-) create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouterIntegrationTest.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouterTest.kt create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/TestMessageConverter.kt diff --git a/README.md b/README.md index 942a55b73..b9a6629b0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (5.4.0) +# th2 common library (Java) (5.4.1) ## Usage @@ -367,6 +367,45 @@ It means that the users can execute calls from the console or through scripts via [grpcurl](https://github.com/fullstorydev/grpcurl#grpcurl) without gRPC schema (files with proto extensions describes gRPC service structure) +## MQ router + +This kind of router provides the ability for component to send / receive messages via RabbitMQ. +Router has several methods to subscribe and publish RabbitMQ messages steam (th2 use batches of messages or events as transport). + +#### Choice pin by attributes + +Pin attributes are key mechanism to choose pin for action execution. Router search all pins which have full set of passed attributes. +For example, the pins: `first` [`publish`, `raw`, `custom_a` ], `second` [`publish`, `raw`, `custom_b` ]. +* only the `first` pin will be chosen by attribut sets: [`custom_a`], [`custom_a`, `raw`], [`custom_a`, `publish`], [`publish`, `raw`, `custom_a` ] +* both pins will be chosen by attribut sets: [`raw`], [`publish`], [`publish`, `raw` ] + +Router implementation and methods have predefined attributes. Result set of attributes for searching pin is union of , , attributes. +Predefined attributes: +* `RabbitMessageGroupBatchRouter` hasn't got any predefined attributes +* `EventBatchRouter` has `evnet` attribute +* `TransportGroupBatchRouter` has `transport-group` attribute + +* `send*` exclude `sendExclusive` methods have `publish` attribute +* `subscribe*` excluded `subscribeExclusive` methods have `subscribe` attribute + +#### Choice publish pin + +Router chooses pins in two stages. At first select all pins matched by attributes than check passed message (batch) by +pin's filters and then send the whole message or its part via pins leaved after two steps. + +#### Choice subscribe pin + +Router chooses pins only by attributes. Pin's filters are used when message has been delivered and parsed. Registered lister doesn't receive message, or it parts failure check by pin's filter. + +### Restrictions: + +Some methods have `All` suffix, it means that developer can publish or subscribe message via 1 or several pins otherwise via only 1 pin. +If number of passed check pins are different then required range, method throw an exception. + +Developer can register only one listener for each pin but one listener can handle messages from several pins. + +`TransportGroupBatchRouter` router doesn't split incoming or outgoing batch by filter not unlike `RabbitMessageGroupBatchRouter` router + ## Export common metrics to Prometheus It can be performed by the following utility methods in CommonMetrics class @@ -455,6 +494,10 @@ dependencies { ## Release notes +### 5.4.1-dev +#### Fix ++ `SubscriberMonitor` is returned from `MessageRouter.subscribe` methods is proxy object to manage RabbitMQ subscribtion without internal listener + ### 5.4.0-dev #### Updated + bom: `4.4.0-dev` to `4.5.0-dev` diff --git a/gradle.properties b/gradle.properties index ff7e89a95..827b6830d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.4.0 +release_version=5.4.1 description='th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j kapt.include.compile.classpath=false diff --git a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java index cd53023c0..9dbc1e937 100644 --- a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchRouter.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -17,6 +17,7 @@ import java.util.Set; +import com.exactpro.th2.common.schema.message.ConfirmationListener; import org.apache.commons.collections4.SetUtils; import org.jetbrains.annotations.NotNull; @@ -70,11 +71,16 @@ protected MessageSender createSender(QueueConfiguration queueConfigu @NotNull @Override - protected MessageSubscriber createSubscriber(QueueConfiguration queueConfiguration, @NotNull String pinName) { + protected MessageSubscriber createSubscriber( + QueueConfiguration queueConfiguration, + @NotNull String pinName, + @NotNull ConfirmationListener listener + ) { return new EventBatchSubscriber( getConnectionManager(), queueConfiguration.getQueue(), - pinName + pinName, + listener ); } diff --git a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java index e05e74250..78642b061 100644 --- a/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/event/EventBatchSubscriber.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ package com.exactpro.th2.common.schema.event; import com.exactpro.th2.common.grpc.EventBatch; +import com.exactpro.th2.common.schema.message.ConfirmationListener; import com.exactpro.th2.common.schema.message.DeliveryMetadata; -import com.exactpro.th2.common.schema.message.FilterFunction; import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitSubscriber; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; @@ -43,9 +43,10 @@ public class EventBatchSubscriber extends AbstractRabbitSubscriber { public EventBatchSubscriber( @NotNull ConnectionManager connectionManager, @NotNull String queue, - @NotNull String th2Pin + @NotNull String th2Pin, + @NotNull ConfirmationListener listener ) { - super(connectionManager, queue, th2Pin, EVENT_TYPE); + super(connectionManager, queue, th2Pin, EVENT_TYPE, listener); } @Override diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java index 739ea0359..e4ce18803 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/AbstractGrpcRouter.java @@ -49,8 +49,6 @@ /** * Abstract implementation for {@link GrpcRouter} *

- * Implement {@link GrpcRouter#init(GrpcRouterConfiguration)} - *

* Implement {@link GrpcRouter#init(GrpcConfiguration, GrpcRouterConfiguration)} *

* Implement {@link GrpcRouter#startServer(BindableService...)} @@ -122,11 +120,6 @@ public abstract class AbstractGrpcRouter implements GrpcRouter { protected static final Map GRPC_RECEIVE_CALL_RESPONSE_SIZE_MAP = new ConcurrentHashMap<>(); - @Override - public void init(GrpcRouterConfiguration configuration) { - init(new GrpcConfiguration(), configuration); - } - @Override public void init(@NotNull GrpcConfiguration configuration, @NotNull GrpcRouterConfiguration routerConfiguration) { failIfInitialized(); diff --git a/src/main/java/com/exactpro/th2/common/schema/grpc/router/GrpcRouter.java b/src/main/java/com/exactpro/th2/common/schema/grpc/router/GrpcRouter.java index 8190ab725..47e959d6f 100644 --- a/src/main/java/com/exactpro/th2/common/schema/grpc/router/GrpcRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/grpc/router/GrpcRouter.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -26,11 +26,6 @@ * @see AbstractGrpcRouter */ public interface GrpcRouter extends AutoCloseable { - /** - * Initialization router - */ - @Deprecated(since = "3.9.0", forRemoval = true) - void init(GrpcRouterConfiguration configuration); void init(@NotNull GrpcConfiguration configuration, @NotNull GrpcRouterConfiguration routerConfiguration); diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java index 60175b9e3..c7c781de9 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageRouter.java @@ -61,28 +61,36 @@ default void init(@NotNull MessageRouterContext context, @NotNull MessageRouter< ExclusiveSubscriberMonitor subscribeExclusive(MessageListener callback); /** - * Listen ONE RabbitMQ queue by intersection schemas queues attributes + * Listen ONE RabbitMQ queue by intersection schemas queues attributes. + * Restrictions: + * You can create only one subscription to th2 pin using any subscribe* functions. + * Internal state: + * Router uses external Connection Manage to interact with RabbitMQ, which holds one connection and one channel per th2 pin in general. + * This rule exception is re-connect to RabbitMQ when the manager establishes new connection and creates new channels. * @param callback listener * @param queueAttr queues attributes * @throws IllegalStateException when more than 1 queue is found - * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + * @throws RuntimeException when the th2 pin is matched by passed attributes already has active subscription + * @return {@link SubscriberMonitor} it start listening. */ SubscriberMonitor subscribe(MessageListener callback, String... queueAttr); /** * Listen SOME RabbitMQ queues by intersection schemas queues attributes + * @see #subscribe(MessageListener, String...) * @param callback listener * @param queueAttr queues attributes - * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + * @return {@link SubscriberMonitor} it start listening. */ SubscriberMonitor subscribeAll(MessageListener callback, String... queueAttr); /** * Listen ONE RabbitMQ queue by intersection schemas queues attributes + * @see #subscribe(MessageListener, String...) * @param queueAttr queues attributes * @param callback listener with manual confirmation * @throws IllegalStateException when more than 1 queue is found - * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + * @return {@link SubscriberMonitor} it start listening. */ default SubscriberMonitor subscribeWithManualAck(ManualConfirmationListener callback, String... queueAttr) { // TODO: probably should not have default implementation @@ -91,9 +99,10 @@ default SubscriberMonitor subscribeWithManualAck(ManualConfirmationListener c /** * Listen SOME RabbitMQ queues by intersection schemas queues attributes + * @see #subscribe(MessageListener, String...) * @param callback listener with manual confirmation * @param queueAttr queues attributes - * @return {@link SubscriberMonitor} it start listening. Returns null if can not listen to this queue + * @return {@link SubscriberMonitor} it start listening. */ default SubscriberMonitor subscribeAllWithManualAck(ManualConfirmationListener callback, String... queueAttr) { // TODO: probably should not have default implementation diff --git a/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java index d234484d2..e4c2feb07 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/MessageSubscriber.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -15,34 +15,10 @@ package com.exactpro.th2.common.schema.message; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; - -import org.jetbrains.annotations.NotNull; - import javax.annotation.concurrent.ThreadSafe; /** * Listen messages and transmit it to {@link MessageListener} */ @ThreadSafe -public interface MessageSubscriber extends AutoCloseable { - // Please use constructor for initialization - @Deprecated(since = "3.3.0", forRemoval = true) - void init(@NotNull ConnectionManager connectionManager, @NotNull String exchangeName, @NotNull SubscribeTarget subscribeTargets); - - // Please use constructor for initialization - @Deprecated - void init(@NotNull ConnectionManager connectionManager, @NotNull SubscribeTarget subscribeTarget, @NotNull FilterFunction filterFunc); - - /** - * @deprecated subscriber automatically subscribe after adding first listener and - * unsubscribe after remove the last one - */ - @Deprecated - void start() throws Exception; - - void addListener(ConfirmationListener messageListener); - - void removeListener(ConfirmationListener messageListener); -} +public interface MessageSubscriber extends AutoCloseable { } diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java index 3547e1853..40e44c94d 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitSubscriber.java @@ -18,11 +18,9 @@ import com.exactpro.th2.common.metrics.HealthMetrics; import com.exactpro.th2.common.schema.message.ConfirmationListener; import com.exactpro.th2.common.schema.message.DeliveryMetadata; -import com.exactpro.th2.common.schema.message.FilterFunction; import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback.Confirmation; import com.exactpro.th2.common.schema.message.MessageSubscriber; import com.exactpro.th2.common.schema.message.SubscriberMonitor; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget; import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager; import com.google.common.base.Suppliers; import com.google.common.io.BaseEncoding; @@ -36,12 +34,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Iterator; import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; import static com.exactpro.th2.common.metrics.CommonMetrics.DEFAULT_BUCKETS; @@ -50,7 +45,7 @@ import static com.exactpro.th2.common.metrics.CommonMetrics.TH2_TYPE_LABEL; import static java.util.Objects.requireNonNull; -public abstract class AbstractRabbitSubscriber implements MessageSubscriber { +public abstract class AbstractRabbitSubscriber implements MessageSubscriber { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRabbitSubscriber.class); @SuppressWarnings("rawtypes") @@ -71,99 +66,44 @@ public abstract class AbstractRabbitSubscriber implements MessageSubscriber> listeners = ConcurrentHashMap.newKeySet(); + private final boolean manualConfirmation; + private final ConfirmationListener listener; private final String queue; private final ConnectionManager connectionManager; private final AtomicReference> consumerMonitor = new AtomicReference<>(emptySupplier()); + private final AtomicBoolean isAlive = new AtomicBoolean(true); private final String th2Type; - private final HealthMetrics healthMetrics = new HealthMetrics(this); + protected final String th2Pin; + public AbstractRabbitSubscriber( @NotNull ConnectionManager connectionManager, @NotNull String queue, @NotNull String th2Pin, - @NotNull String th2Type + @NotNull String th2Type, + @NotNull ConfirmationListener listener ) { this.connectionManager = requireNonNull(connectionManager, "Connection can not be null"); this.queue = requireNonNull(queue, "Queue can not be null"); - this.th2Pin = requireNonNull(th2Pin, "TH2 pin can not be null"); - this.th2Type = requireNonNull(th2Type, "TH2 type can not be null"); + this.th2Pin = requireNonNull(th2Pin, "th2 pin can not be null"); + this.th2Type = requireNonNull(th2Type, "th2 type can not be null"); + this.listener = requireNonNull(listener, "Listener can not be null"); + this.manualConfirmation = ConfirmationListener.isManual(listener); + subscribe(); } - @Deprecated @Override - public void init(@NotNull ConnectionManager connectionManager, @NotNull String exchangeName, @NotNull SubscribeTarget subscribeTargets) { - throw new UnsupportedOperationException("Method is deprecated, please use constructor"); - } - - @Deprecated - @Override - public void init(@NotNull ConnectionManager connectionManager, @NotNull SubscribeTarget subscribeTarget, @NotNull FilterFunction filterFunc) { - throw new UnsupportedOperationException("Method is deprecated, please use constructor"); - } - - @Override - @Deprecated - public void start() throws Exception { - // Do nothing - } - - @Override - public void addListener(ConfirmationListener messageListener) { - publicLock.lock(); - try { - boolean isManual = ConfirmationListener.isManual(messageListener); - if (isManual && hasManualSubscriber) { - throw new IllegalStateException("cannot subscribe listener " + messageListener - + " because only one listener with manual confirmation is allowed per queue"); - } - if(listeners.add(messageListener)) { - hasManualSubscriber |= isManual; - subscribe(); - } - } finally { - publicLock.unlock(); - } - } - - @Override - public void removeListener(ConfirmationListener messageListener) { - publicLock.lock(); - try { - boolean isManual = ConfirmationListener.isManual(messageListener); - if (listeners.remove(messageListener)) { - hasManualSubscriber &= !isManual; - messageListener.onClose(); - } - } finally { - publicLock.unlock(); + public void close() throws IOException { + if (!isAlive.getAndSet(false)) { + LOGGER.warn("Subscriber for '{}' pin is already closed", th2Pin); + return; } - } - @Override - public void close() throws IOException { - publicLock.lock(); - try { - SubscriberMonitor monitor = consumerMonitor.getAndSet(emptySupplier()).get(); - if (monitor != null) { - monitor.unsubscribe(); - } + SubscriberMonitor monitor = consumerMonitor.getAndSet(emptySupplier()).get(); + monitor.unsubscribe(); - Iterator> listIterator = listeners.iterator(); - while (listIterator.hasNext()) { - ConfirmationListener listener = listIterator.next(); - listIterator.remove(); - listener.onClose(); - } - } finally { - publicLock.unlock(); - } + listener.onClose(); } protected abstract T valueFromBytes(byte[] body) throws Exception; @@ -194,18 +134,12 @@ protected void handle(DeliveryMetadata deliveryMetadata, Delivery delivery, T va return; } - boolean hasManualConfirmation = false; - for (ConfirmationListener listener : listeners) { - try { - listener.handle(deliveryMetadata, filteredValue, confirmation); - if (!hasManualConfirmation) { - hasManualConfirmation = ConfirmationListener.isManual(listener); - } - } catch (Exception listenerExc) { - LOGGER.warn("Message listener from class '{}' threw exception", listener.getClass(), listenerExc); - } + try { + listener.handle(deliveryMetadata, filteredValue, confirmation); + } catch (Exception listenerExc) { + LOGGER.warn("Message listener from class '{}' threw exception", listener.getClass(), listenerExc); } - if (!hasManualConfirmation) { + if (!manualConfirmation) { confirmation.confirm(); } } catch (Exception e) { @@ -227,6 +161,10 @@ private void subscribe() { private void resubscribe() { LOGGER.info("Try to resubscribe subscriber for queue name='{}'", queue); + if (!isAlive.get()) { + LOGGER.warn("Subscriber for '{}' pin is already closed", th2Pin); + return; + } SubscriberMonitor monitor = consumerMonitor.getAndSet(emptySupplier()).get(); if (monitor != null) { diff --git a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java index 08e7cb888..e1564aff8 100644 --- a/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java +++ b/src/main/java/com/exactpro/th2/common/schema/message/impl/rabbitmq/connection/ConnectionManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt index e844452b3..2cefaa325 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/DeliveryMetadata.kt @@ -1,5 +1,5 @@ /* - * Copyright 2022 Exactpro (Exactpro Systems Limited) + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt index 511ab4507..911624556 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouter.kt @@ -49,7 +49,7 @@ abstract class AbstractRabbitRouter : MessageRouter { private val boxConfiguration: BoxConfiguration get() = context.boxConfiguration - private val subscribers = ConcurrentHashMap>() + private val subscribers = ConcurrentHashMap() private val senders = ConcurrentHashMap>() override fun init(context: MessageRouterContext) { @@ -85,16 +85,14 @@ abstract class AbstractRabbitRouter : MessageRouter { val queue = connectionManager.queueDeclare() val pinConfig = PinConfiguration("", queue, "", isReadable = true, isWritable = false) - val messageListener = ConfirmationListener.wrap(callback) - val messageSubscriber = subscribers.getSubscriber(queue, pinConfig).apply { - addListener(messageListener) - } + val listener = ConfirmationListener.wrap(callback) + subscribers.registerSubscriber(queue, pinConfig, listener) return object: ExclusiveSubscriberMonitor { override val queue: String = queue override fun unsubscribe() { - messageSubscriber.removeListener(messageListener) + subscribers.unregisterSubscriber(queue, pinConfig) } } } @@ -172,7 +170,11 @@ abstract class AbstractRabbitRouter : MessageRouter { protected abstract fun createSender(pinConfig: PinConfiguration, pinName: PinName, bookName: BookName): MessageSender //TODO: implement common subscriber - protected abstract fun createSubscriber(pinConfig: PinConfiguration, pinName: PinName): MessageSubscriber + protected abstract fun createSubscriber( + pinConfig: PinConfiguration, + pinName: PinName, + listener: ConfirmationListener + ): MessageSubscriber protected abstract fun T.toErrorString(): String @@ -207,7 +209,7 @@ abstract class AbstractRabbitRouter : MessageRouter { private fun subscribe( pintAttributes: Set, - messageListener: ConfirmationListener, + listener: ConfirmationListener, check: List.() -> Unit ): SubscriberMonitor { val packages: List = configuration.queues.asSequence() @@ -221,14 +223,12 @@ abstract class AbstractRabbitRouter : MessageRouter { val monitors: MutableList = mutableListOf() packages.forEach { (pinName: PinName, pinConfig: PinConfiguration) -> runCatching { - subscribers.getSubscriber(pinName, pinConfig).apply { - addListener(messageListener) - } + subscribers.registerSubscriber(pinName, pinConfig, listener) }.onFailure { e -> LOGGER.error(e) { "Listener can't be subscribed via the $pinName pin" } exceptions[pinName] = e - }.onSuccess { messageSubscriber -> - monitors.add(SubscriberMonitor { messageSubscriber.removeListener(messageListener) }) + }.onSuccess { + monitors.add(SubscriberMonitor { subscribers.unregisterSubscriber(pinName, pinConfig) }) } } @@ -261,15 +261,33 @@ abstract class AbstractRabbitRouter : MessageRouter { return@computeIfAbsent createSender(pinConfig, pinName, boxConfiguration.bookName) } - private fun ConcurrentHashMap>.getSubscriber( + private fun ConcurrentHashMap.registerSubscriber( pinName: PinName, - pinConfig: PinConfiguration - ): MessageSubscriber = computeIfAbsent(pinConfig.queue) { - check(pinConfig.isReadable) { - "The $pinName isn't readable, configuration: $pinConfig" + pinConfig: PinConfiguration, + listener: ConfirmationListener + ) { + compute(pinConfig.queue) { _, previous -> + check(previous == null) { + "The '$pinName' pin already has subscriber, configuration: $pinConfig" + } + check(pinConfig.isReadable) { + "The $pinName isn't readable, configuration: $pinConfig" + } + + createSubscriber(pinConfig, pinName, listener).also { + LOGGER.info { "Created subscriber for '$pinName' pin, configuration: $pinConfig" } + } } + } - return@computeIfAbsent createSubscriber(pinConfig, pinName) + private fun ConcurrentHashMap.unregisterSubscriber( + pinName: PinName, + pinConfig: PinConfiguration, + ) { + remove(pinConfig.queue)?.let { subscriber -> + subscriber.close() + LOGGER.info { "Removed subscriber for '$pinName' pin, configuration: $pinConfig" } + } } private open class PinInfo( diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt index 832c64a4c..323c88887 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/custom/RabbitCustomRouter.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -15,6 +15,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.custom +import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.QueueAttribute @@ -70,13 +71,18 @@ class RabbitCustomRouter( ) } - override fun createSubscriber(pinConfig: PinConfiguration, pinName: PinName): MessageSubscriber { + override fun createSubscriber( + pinConfig: PinConfiguration, + pinName: PinName, + listener: ConfirmationListener + ): MessageSubscriber { return Subscriber( connectionManager, pinConfig.queue, pinName, customTag, - converter + converter, + listener ) } @@ -105,8 +111,9 @@ class RabbitCustomRouter( queue: String, th2Pin: String, customTag: String, - private val converter: MessageConverter - ) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, customTag) { + private val converter: MessageConverter, + messageListener: ConfirmationListener + ) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, customTag, messageListener) { override fun valueFromBytes(body: ByteArray): T = converter.fromByteArray(body) override fun toShortTraceString(value: T): String = converter.toTraceString(value) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt index 23c0e6d25..d6cce606a 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt @@ -18,12 +18,14 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.group import com.exactpro.th2.common.grpc.MessageGroupBatch import com.exactpro.th2.common.metrics.* import com.exactpro.th2.common.schema.filter.strategy.impl.AnyMessageFilterStrategy +import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.configuration.RouterFilter import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName +import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName import com.exactpro.th2.common.schema.message.toBuilderWithMetadata import com.google.protobuf.Message @@ -74,16 +76,18 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter() } override fun createSubscriber( - pinConfig: QueueConfiguration, - pinName: PinName - ): MessageSubscriber { + pinConfig: PinConfiguration, + pinName: PinName, + listener: ConfirmationListener + ): MessageSubscriber { return RabbitMessageGroupBatchSubscriber( connectionManager, pinConfig.queue, { msg: Message, filters: List -> AnyMessageFilterStrategy.verify(msg, filters) }, pinName, pinConfig.filters, - connectionManager.configuration.messageRecursionLimit + connectionManager.configuration.messageRecursionLimit, + listener ) } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt index fcffae922..194233f01 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchSubscriber.kt @@ -23,6 +23,7 @@ import com.exactpro.th2.common.metrics.SESSION_ALIAS_LABEL import com.exactpro.th2.common.metrics.TH2_PIN_LABEL import com.exactpro.th2.common.metrics.incrementDroppedMetrics import com.exactpro.th2.common.metrics.incrementTotalMetrics +import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.FilterFunction import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback @@ -45,8 +46,9 @@ class RabbitMessageGroupBatchSubscriber( private val filterFunction: FilterFunction, th2Pin: String, private val filters: List, - private val messageRecursionLimit: Int -) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, MESSAGE_GROUP_TYPE) { + private val messageRecursionLimit: Int, + messageListener: ConfirmationListener +) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, MESSAGE_GROUP_TYPE, messageListener) { private val logger = KotlinLogging.logger {} override fun valueFromBytes(body: ByteArray): MessageGroupBatch = parseEncodedBatch(body) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt index 114e0abc8..f8b66e930 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/notification/NotificationEventBatchSubscriber.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,45 +19,23 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.notification import com.exactpro.th2.common.grpc.EventBatch import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.DeliveryMetadata -import com.exactpro.th2.common.schema.message.FilterFunction import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.SubscriberMonitor -import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.SubscribeTarget import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager import com.rabbitmq.client.Delivery -import java.util.concurrent.CopyOnWriteArrayList import mu.KotlinLogging +import java.util.concurrent.CopyOnWriteArrayList // DRAFT of notification router class NotificationEventBatchSubscriber( private val connectionManager: ConnectionManager, private val queue: String -) : MessageSubscriber { +) : MessageSubscriber { private val listeners = CopyOnWriteArrayList>() private lateinit var monitor: SubscriberMonitor - @Deprecated( - "Method is deprecated, please use constructor", - ReplaceWith("NotificationEventBatchSubscriber()") - ) - override fun init(connectionManager: ConnectionManager, exchangeName: String, subscribeTargets: SubscribeTarget) { - throw UnsupportedOperationException("Method is deprecated, please use constructor") - } - - @Deprecated( - "Method is deprecated, please use constructor", - ReplaceWith("NotificationEventBatchSubscriber()") - ) - override fun init( - connectionManager: ConnectionManager, - subscribeTarget: SubscribeTarget, - filterFunc: FilterFunction - ) { - throw UnsupportedOperationException("Method is deprecated, please use constructor") - } - - override fun start() { + fun start() { monitor = connectionManager.basicConsume( queue, { deliveryMetadata: DeliveryMetadata, delivery: Delivery, confirmation: ManualAckDeliveryCallback.Confirmation -> @@ -81,11 +59,11 @@ class NotificationEventBatchSubscriber( ) } - override fun removeListener(messageListener: ConfirmationListener) { + fun removeListener(messageListener: ConfirmationListener) { listeners.remove(messageListener) } - override fun addListener(messageListener: ConfirmationListener) { + fun addListener(messageListener: ConfirmationListener) { listeners.add(messageListener) } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouter.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouter.kt index 6d4d62833..a887fc063 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouter.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouter.kt @@ -15,12 +15,14 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport +import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.MessageSender import com.exactpro.th2.common.schema.message.MessageSubscriber import com.exactpro.th2.common.schema.message.QueueAttribute import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.AbstractRabbitRouter import com.exactpro.th2.common.schema.message.impl.rabbitmq.BookName +import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.PinName import org.jetbrains.annotations.NotNull @@ -46,14 +48,16 @@ class TransportGroupBatchRouter : AbstractRabbitRouter() { } override fun createSubscriber( - pinConfig: QueueConfiguration, + pinConfig: PinConfiguration, pinName: PinName, - ): MessageSubscriber { + listener: ConfirmationListener + ): MessageSubscriber { return TransportGroupBatchSubscriber( connectionManager, pinConfig.queue, pinName, - pinConfig.filters + pinConfig.filters, + listener ) } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSubscriber.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSubscriber.kt index 2c996a986..4cc75b9eb 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSubscriber.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchSubscriber.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -18,6 +18,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.exactpro.th2.common.metrics.BOOK_NAME_LABEL import com.exactpro.th2.common.metrics.SESSION_GROUP_LABEL import com.exactpro.th2.common.metrics.TH2_PIN_LABEL +import com.exactpro.th2.common.schema.message.ConfirmationListener import com.exactpro.th2.common.schema.message.DeliveryMetadata import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback import com.exactpro.th2.common.schema.message.configuration.RouterFilter @@ -33,7 +34,8 @@ class TransportGroupBatchSubscriber( queue: String, th2Pin: String, private val filters: List, -) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, TRANSPORT_GROUP_TYPE) { + messageListener: ConfirmationListener +) : AbstractRabbitSubscriber(connectionManager, queue, th2Pin, TRANSPORT_GROUP_TYPE, messageListener) { override fun valueFromBytes(body: ByteArray): GroupBatch = Unpooled.wrappedBuffer(body).run(GroupBatchCodec::decode) diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt index 8d1e2ad80..3bc849e95 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/TestConfirmationMessageListenerWrapper.kt @@ -1,5 +1,5 @@ /* - * Copyright 2022 Exactpro (Exactpro Systems Limited) + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouterIntegrationTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouterIntegrationTest.kt new file mode 100644 index 000000000..d432f2ccc --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouterIntegrationTest.kt @@ -0,0 +1,226 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.message.impl.rabbitmq + +import com.exactpro.th2.common.annotations.IntegrationTest +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration +import com.exactpro.th2.common.schema.message.ManualAckDeliveryCallback +import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration +import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration +import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.RabbitCustomRouter +import com.rabbitmq.client.BuiltinExchangeType +import mu.KotlinLogging +import org.junit.jupiter.api.Assertions.assertNull +import org.mockito.kotlin.mock +import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName +import java.time.Duration +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.TimeUnit +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +@IntegrationTest +class AbstractRabbitRouterIntegrationTest { + + @Test + fun `receive unconfirmed message after resubscribe`() { + RabbitMQContainer(DockerImageName.parse(RABBITMQ_MANAGEMENT_ALPINE)) + .withExchange(EXCHANGE, BuiltinExchangeType.DIRECT.type, false, false, true, emptyMap()) + .withQueue(QUEUE_NAME, false, true, emptyMap()) + .withBinding( + EXCHANGE, + QUEUE_NAME, emptyMap(), + ROUTING_KEY, "queue" + ) + .use { rabbitMQContainer -> + rabbitMQContainer.start() + K_LOGGER.info { "Started with port ${rabbitMQContainer.amqpPort}, rest ${rabbitMQContainer.httpUrl} ${rabbitMQContainer.adminUsername} ${rabbitMQContainer.adminPassword} " } + + createConnectionManager(rabbitMQContainer).use { firstManager -> + createRouter(firstManager).use { firstRouter -> + val messageA = "test-message-a" + val messageB = "test-message-b" + val messageC = "test-message-c" + val messageD = "test-message-d" + + val queue = ArrayBlockingQueue(4) + + firstRouter.send(messageA) + firstRouter.send(messageB) + firstRouter.send(messageC) + firstRouter.send(messageD) + + connectAndCheck( + rabbitMQContainer, queue, listOf( + Expectation(messageA, false, ManualAckDeliveryCallback.Confirmation::confirm), + Expectation(messageB, false, ManualAckDeliveryCallback.Confirmation::reject), + Expectation(messageC, false) { }, + Expectation(messageD, false) { }, + ) + ) + + connectAndCheck( + rabbitMQContainer, queue, listOf( + Expectation(messageC, true, ManualAckDeliveryCallback.Confirmation::confirm), + Expectation(messageD, true) { }, + ) + ) + + connectAndCheck( + rabbitMQContainer, queue, listOf( + Expectation(messageD, true, ManualAckDeliveryCallback.Confirmation::reject), + ) + ) + + connectAndCheck(rabbitMQContainer, queue, emptyList()) + } + } + } + } + + private fun connectAndCheck( + rabbitMQContainer: RabbitMQContainer, + queue: ArrayBlockingQueue, + expectations: List, + ) { + createConnectionManager(rabbitMQContainer).use { manager -> + createRouter(manager).use { router -> + val monitor = router.subscribeWithManualAck({ deliveryMetadata, message, confirmation -> + queue.put( + Delivery( + message, + deliveryMetadata.isRedelivered, + confirmation + ) + ) + }) + + try { + expectations.forEach { expectation -> + val delivery = assertNotNull(queue.poll(1, TimeUnit.SECONDS)) + assertEquals(expectation.message, delivery.message, "Message") + assertEquals(expectation.redelivery, delivery.redelivery, "Redelivery flag") + expectation.action.invoke(delivery.confirmation) + } + + assertNull(queue.poll(1, TimeUnit.SECONDS)) + } finally { + monitor.unsubscribe() + } + } + + createRouter(manager).use { router -> + val monitor = router.subscribeWithManualAck({ deliveryMetadata, message, confirmation -> + queue.put( + Delivery( + message, + deliveryMetadata.isRedelivered, + confirmation + ) + ) + }) + + try { + // RabbitMQ doesn't resend messages after resubscribe using the same connection and channel + assertNull(queue.poll(1, TimeUnit.SECONDS)) + } finally { + monitor.unsubscribe() + } + } + } + } + + private fun createConnectionManager( + rabbitMQContainer: RabbitMQContainer, + prefetchCount: Int = DEFAULT_PREFETCH_COUNT, + confirmationTimeout: Duration = DEFAULT_CONFIRMATION_TIMEOUT + ) = ConnectionManager( + RabbitMQConfiguration( + host = rabbitMQContainer.host, + vHost = "", + port = rabbitMQContainer.amqpPort, + username = rabbitMQContainer.adminUsername, + password = rabbitMQContainer.adminPassword, + ), + ConnectionManagerConfiguration( + subscriberName = "test", + prefetchCount = prefetchCount, + confirmationTimeout = confirmationTimeout, + ), + ) { + K_LOGGER.error { "Fatal connection problem" } + } + + private fun createRouter(connectionManager: ConnectionManager) = RabbitCustomRouter( + "test-custom-tag", + arrayOf("test-label"), + TestMessageConverter() + ).apply { + init( + DefaultMessageRouterContext( + connectionManager, + mock { }, + MessageRouterConfiguration( + mapOf( + "test" to QueueConfiguration( + routingKey = ROUTING_KEY, + queue = "", + exchange = "test-exchange", + attributes = listOf("publish") + ), + "test1" to QueueConfiguration( + routingKey = "", + queue = QUEUE_NAME, + exchange = EXCHANGE, + attributes = listOf("subscribe") + ), + ) + ), + BoxConfiguration() + ) + ) + } + + companion object { + private val K_LOGGER = KotlinLogging.logger { } + + private const val RABBITMQ_MANAGEMENT_ALPINE = "rabbitmq:3.11.2-management-alpine" + private const val ROUTING_KEY = "routingKey" + private const val QUEUE_NAME = "queue" + private const val EXCHANGE = "test-exchange" + + private const val DEFAULT_PREFETCH_COUNT = 10 + private val DEFAULT_CONFIRMATION_TIMEOUT: Duration = Duration.ofSeconds(1) + + private class Expectation( + val message: String, + val redelivery: Boolean, + val action: ManualAckDeliveryCallback.Confirmation.() -> Unit + ) + + private class Delivery( + val message: String, + val redelivery: Boolean, + val confirmation: ManualAckDeliveryCallback.Confirmation + ) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouterTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouterTest.kt new file mode 100644 index 000000000..e5fa77b6d --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/AbstractRabbitRouterTest.kt @@ -0,0 +1,264 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.message.impl.rabbitmq + +import com.exactpro.th2.common.event.bean.BaseTest +import com.exactpro.th2.common.schema.message.ExclusiveSubscriberMonitor +import com.exactpro.th2.common.schema.message.configuration.GlobalNotificationConfiguration +import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration +import com.exactpro.th2.common.schema.message.configuration.QueueConfiguration +import com.exactpro.th2.common.schema.message.impl.context.DefaultMessageRouterContext +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager +import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.RabbitCustomRouter +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertAll +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.function.Executable +import org.mockito.kotlin.any +import org.mockito.kotlin.argThat +import org.mockito.kotlin.clearInvocations +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.mockito.kotlin.verifyNoMoreInteractions +import java.util.concurrent.atomic.AtomicInteger +import kotlin.test.assertNotEquals + +private const val TEST_EXCLUSIVE_QUEUE = "test-exclusive-queue" + +class AbstractRabbitRouterTest { + private val connectionConfiguration = ConnectionManagerConfiguration() + private val managerMonitor: ExclusiveSubscriberMonitor = mock { } + private val connectionManager: ConnectionManager = mock { + on { configuration }.thenReturn(connectionConfiguration) + on { basicConsume(any(), any(), any()) }.thenReturn(managerMonitor) + on { queueDeclare() }.thenAnswer { "$TEST_EXCLUSIVE_QUEUE-${exclusiveQueueCounter.incrementAndGet()}" } + } + + @Nested + inner class Subscribing { + private val router = createRouter( + mapOf( + "test" to QueueConfiguration( + routingKey = "publish", + queue = "", + exchange = "test-exchange", + attributes = listOf("publish", "test") + ), + "test1" to QueueConfiguration( + routingKey = "", + queue = "queue1", + exchange = "test-exchange", + attributes = listOf("subscribe", "1") + ), + "test2" to QueueConfiguration( + routingKey = "", + queue = "queue2", + exchange = "test-exchange", + attributes = listOf("subscribe", "2") + ) + ) + ) + + @AfterEach + fun afterEach() { + verifyNoMoreInteractions(connectionManager) + verifyNoMoreInteractions(managerMonitor) + } + + @Test + fun `subscribes to correct queue`() { + val monitor = router.subscribe(mock { }, "1") + assertNotNull(monitor) { "monitor must not be null" } + + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + } + + @Test + fun `subscribes with manual ack to correct queue`() { + val monitor = router.subscribeWithManualAck(mock { }, "1") + assertNotNull(monitor) { "monitor must not be null" } + + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + } + + @Test + fun `subscribes to all matched queues`() { + val monitor = router.subscribeAll(mock { }) + assertNotNull(monitor) { "monitor must not be null" } + + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + verify(connectionManager).basicConsume(eq("queue2"), any(), any()) + } + + @Test + fun `unsubscribe after subscribe`() { + val monitor = router.subscribe(mock { }, "1") + clearInvocations(connectionManager) + clearInvocations(managerMonitor) + + monitor.unsubscribe() + verify(managerMonitor).unsubscribe() + } + + @Test + fun `unsubscribe after subscribe with manual ack`() { + val monitor = router.subscribeWithManualAck(mock { }, "1") + clearInvocations(connectionManager) + clearInvocations(managerMonitor) + + monitor.unsubscribe() + verify(managerMonitor).unsubscribe() + } + + @Test + fun `unsubscribe after subscribe to exclusive queue`() { + val monitor = router.subscribeExclusive(mock { }) + clearInvocations(connectionManager) + clearInvocations(managerMonitor) + + monitor.unsubscribe() + verify(managerMonitor).unsubscribe() + } + + @Test + fun `subscribes after unsubscribe`() { + router.subscribe(mock { }, "1") + .unsubscribe() + clearInvocations(connectionManager) + clearInvocations(managerMonitor) + + val monitor = router.subscribe(mock { }, "1") + assertNotNull(monitor) { "monitor must not be null" } + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + } + + @Test + fun `subscribes with manual ack after unsubscribe`() { + router.subscribeWithManualAck(mock { }, "1") + .unsubscribe() + clearInvocations(connectionManager) + clearInvocations(managerMonitor) + + val monitor = router.subscribeWithManualAck(mock { }, "1") + assertNotNull(monitor) { "monitor must not be null" } + verify(connectionManager).basicConsume(eq("queue1"), any(), any()) + } + + @Test + fun `subscribes when subscribtion active`() { + router.subscribe(mock { }, "1") + clearInvocations(connectionManager) + clearInvocations(managerMonitor) + + assertThrows(RuntimeException::class.java) { router.subscribe(mock { }, "1") } + } + + @Test + fun `subscribes with manual ack when subscribtion active`() { + router.subscribeWithManualAck(mock { }, "1") + clearInvocations(connectionManager) + clearInvocations(managerMonitor) + + assertThrows(RuntimeException::class.java) { router.subscribeWithManualAck(mock { }, "1") } + } + + @Test + fun `subscribes to exclusive queue when subscribtion active`() { + val monitorA = router.subscribeExclusive(mock { }) + val monitorB = router.subscribeExclusive(mock { }) + + verify(connectionManager, times(2)).queueDeclare() + verify(connectionManager, times(2)).basicConsume( + argThat { matches(Regex("$TEST_EXCLUSIVE_QUEUE-\\d+")) }, + any(), + any() + ) + assertNotEquals(monitorA.queue, monitorB.queue) + } + + @Test + fun `reports if more that one queue matches`() { + assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }) } + .apply { + assertEquals( + "Found incorrect number of pins [test1, test2] to subscribe operation by attributes [subscribe] and filters, expected 1, actual 2", + message + ) + } + } + + @Test + fun `reports if no queue matches`() { + assertAll( + Executable { + assertThrows(IllegalStateException::class.java) { + router.subscribe( + mock { }, + "unexpected" + ) + } + .apply { + assertEquals( + "Found incorrect number of pins [] to subscribe operation by attributes [unexpected, subscribe] and filters, expected 1, actual 0", + message + ) + } + }, + Executable { + assertThrows(IllegalStateException::class.java) { + router.subscribeAll( + mock { }, + "unexpected" + ) + } + .apply { + assertEquals( + "Found incorrect number of pins [] to subscribe all operation by attributes [unexpected, subscribe] and filters, expected 1 or more, actual 0", + message + ) + } + } + ) + } + } + + private fun createRouter(pins: Map): RabbitCustomRouter = + RabbitCustomRouter( + "test-custom-tag", + arrayOf("test-label"), + TestMessageConverter() + ).apply { + init( + DefaultMessageRouterContext( + connectionManager, + mock { }, + MessageRouterConfiguration(pins, GlobalNotificationConfiguration()), + BaseTest.BOX_CONFIGURATION + ) + ) + } + + companion object { + private val exclusiveQueueCounter = AtomicInteger(0) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/TestMessageConverter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/TestMessageConverter.kt new file mode 100644 index 000000000..8fba22465 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/TestMessageConverter.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.message.impl.rabbitmq + +import com.exactpro.th2.common.schema.message.impl.rabbitmq.custom.MessageConverter + +class TestMessageConverter: MessageConverter { + override fun toByteArray(value: String): ByteArray = value.toByteArray() + + override fun fromByteArray(data: ByteArray): String = String(data) + + override fun extractCount(value: String): Int = 1 + + override fun toDebugString(value: String): String = value + + override fun toTraceString(value: String): String = value +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageBatchRouter.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageBatchRouter.kt index 351a43b35..e6840fec6 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageBatchRouter.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/group/TestRabbitMessageBatchRouter.kt @@ -36,6 +36,7 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.Connec import com.exactpro.th2.common.schema.message.impl.rabbitmq.connection.ConnectionManager import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.function.Executable @@ -45,14 +46,15 @@ import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.eq import org.mockito.kotlin.mock import org.mockito.kotlin.never +import org.mockito.kotlin.times import org.mockito.kotlin.verify class TestRabbitMessageGroupBatchRouter { private val connectionConfiguration = ConnectionManagerConfiguration() - private val monitor: ExclusiveSubscriberMonitor = mock { } + private val managerMonitor: ExclusiveSubscriberMonitor = mock { } private val connectionManager: ConnectionManager = mock { on { configuration }.thenReturn(connectionConfiguration) - on { basicConsume(any(), any(), any()) }.thenReturn(monitor) + on { basicConsume(any(), any(), any()) }.thenReturn(managerMonitor) } @Nested @@ -151,7 +153,7 @@ class TestRabbitMessageGroupBatchRouter { @Test fun `reports about extra pins matches the publication`() { - Assertions.assertThrows(IllegalStateException::class.java) { + assertThrows(IllegalStateException::class.java) { router.send(MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() .apply { this += message(BOOK_NAME, "test-message", Direction.FIRST, "test-alias") } @@ -166,7 +168,7 @@ class TestRabbitMessageGroupBatchRouter { @Test fun `reports about no pins matches the publication`() { - Assertions.assertThrows(IllegalStateException::class.java) { + assertThrows(IllegalStateException::class.java) { router.send(MessageGroupBatch.newBuilder() .addGroups(MessageGroup.newBuilder() .apply { this += message(BOOK_NAME, "test-message", Direction.FIRST, "test-alias") } @@ -256,33 +258,9 @@ class TestRabbitMessageGroupBatchRouter { } } - @Test - fun `subscribes to correct queue`() { - val monitor = router.subscribe(mock { }, "1") - Assertions.assertNotNull(monitor) { "monitor must not be null" } - - verify(connectionManager).basicConsume(eq("queue1"), any(), any()) - } - @Test - fun `subscribes with manual ack to correct queue`() { - val monitor = router.subscribeWithManualAck(mock { }, "1") - Assertions.assertNotNull(monitor) { "monitor must not be null" } - - verify(connectionManager).basicConsume(eq("queue1"), any(), any()) - } - - @Test - fun `subscribes to all matched queues`() { - val monitor = router.subscribeAll(mock { }) - Assertions.assertNotNull(monitor) { "monitor must not be null" } - - verify(connectionManager).basicConsume(eq("queue1"), any(), any()) - verify(connectionManager).basicConsume(eq("queue2"), any(), any()) - } - @Test fun `reports if more that one queue matches`() { - Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }) } + assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }) } .apply { Assertions.assertEquals( "Found incorrect number of pins [test1, test2] to subscribe operation by attributes [subscribe] and filters, expected 1, actual 2", @@ -295,7 +273,7 @@ class TestRabbitMessageGroupBatchRouter { fun `reports if no queue matches`() { Assertions.assertAll( Executable { - Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }, "unexpected") } + assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }, "unexpected") } .apply { Assertions.assertEquals( "Found incorrect number of pins [] to subscribe operation by attributes [unexpected, subscribe] and filters, expected 1, actual 0", @@ -304,7 +282,7 @@ class TestRabbitMessageGroupBatchRouter { } }, Executable { - Assertions.assertThrows(IllegalStateException::class.java) { router.subscribeAll(mock { }, "unexpected") } + assertThrows(IllegalStateException::class.java) { router.subscribeAll(mock { }, "unexpected") } .apply { Assertions.assertEquals( "Found incorrect number of pins [] to subscribe all operation by attributes [unexpected, subscribe] and filters, expected 1 or more, actual 0", diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt index 25fcd7976..27987afc2 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/TransportGroupBatchRouterTest.kt @@ -28,6 +28,8 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportG import io.netty.buffer.Unpooled import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.function.Executable @@ -36,10 +38,10 @@ import java.time.Instant class TransportGroupBatchRouterTest { private val connectionConfiguration = ConnectionManagerConfiguration() - private val monitor: ExclusiveSubscriberMonitor = mock { } + private val managerMonitor: ExclusiveSubscriberMonitor = mock { } private val connectionManager: ConnectionManager = mock { on { configuration }.thenReturn(connectionConfiguration) - on { basicConsume(any(), any(), any()) }.thenReturn(monitor) + on { basicConsume(any(), any(), any()) }.thenReturn(managerMonitor) } @Nested @@ -125,7 +127,7 @@ class TransportGroupBatchRouterTest { @Test fun `reports about extra pins matches the publication`() { - Assertions.assertThrows(IllegalStateException::class.java) { + assertThrows(IllegalStateException::class.java) { router.send(createGroupBatch("test-message")) }.apply { Assertions.assertEquals( @@ -137,7 +139,7 @@ class TransportGroupBatchRouterTest { @Test fun `reports about no pins matches the publication`() { - Assertions.assertThrows(IllegalStateException::class.java) { + assertThrows(IllegalStateException::class.java) { router.send(TRANSPORT_BATCH, "unexpected") }.apply { Assertions.assertEquals( @@ -211,34 +213,9 @@ class TransportGroupBatchRouterTest { } } - @Test - fun `subscribes to correct queue`() { - val monitor = router.subscribe(mock { }, "1") - Assertions.assertNotNull(monitor) { "monitor must not be null" } - - verify(connectionManager).basicConsume(eq("queue1"), any(), any()) - } - - @Test - fun `subscribes with manual ack to correct queue`() { - val monitor = router.subscribeWithManualAck(mock { }, "1") - Assertions.assertNotNull(monitor) { "monitor must not be null" } - - verify(connectionManager).basicConsume(eq("queue1"), any(), any()) - } - - @Test - fun `subscribes to all matched queues`() { - val monitor = router.subscribeAll(mock { }) - Assertions.assertNotNull(monitor) { "monitor must not be null" } - - verify(connectionManager).basicConsume(eq("queue1"), any(), any()) - verify(connectionManager).basicConsume(eq("queue2"), any(), any()) - } - @Test fun `reports if more that one queue matches`() { - Assertions.assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }) } + assertThrows(IllegalStateException::class.java) { router.subscribe(mock { }) } .apply { Assertions.assertEquals( "Found incorrect number of pins [test1, test2] to subscribe operation by attributes [subscribe, $TRANSPORT_GROUP_ATTRIBUTE] and filters, expected 1, actual 2", @@ -251,7 +228,7 @@ class TransportGroupBatchRouterTest { fun `reports if no queue matches`() { Assertions.assertAll( Executable { - Assertions.assertThrows(IllegalStateException::class.java) { + assertThrows(IllegalStateException::class.java) { router.subscribe( mock { }, "unexpected" @@ -265,7 +242,7 @@ class TransportGroupBatchRouterTest { } }, Executable { - Assertions.assertThrows(IllegalStateException::class.java) { + assertThrows(IllegalStateException::class.java) { router.subscribeAll( mock { }, "unexpected" From 4745640872fd75965b5784552078d0d3888ed345 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Thu, 7 Sep 2023 12:15:58 +0400 Subject: [PATCH 150/154] [TH2-5051] Disable dates serialization as timestamps (#276) The default serialization format for LocalData, Time, DateTime is an array of its components. This was disabled. Now they are serialized as they `toString` representation --- README.md | 9 ++++- gradle.properties | 2 +- .../message/impl/rabbitmq/transport/Codecs.kt | 4 ++ .../impl/rabbitmq/transport/CodecsTest.kt | 40 +++++++++++++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b9a6629b0..8929e98a9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (5.4.1) +# th2 common library (Java) (5.4.2) ## Usage @@ -494,6 +494,13 @@ dependencies { ## Release notes +### 5.4.2-dev + +#### Fix + ++ The serialization of `LocalTime`, `LocalDate` and `LocalDateTime` instances corrected for th2 transport parsed message. + Old result would look like `[2023,9,7]`. Corrected serialization result looks like `2023-09-07` + ### 5.4.1-dev #### Fix + `SubscriberMonitor` is returned from `MessageRouter.subscribe` methods is proxy object to manage RabbitMQ subscribtion without internal listener diff --git a/gradle.properties b/gradle.properties index 827b6830d..b876306b8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.4.1 +release_version=5.4.2 description='th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j kapt.include.compile.classpath=false diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt index 0e43f0c65..5fbd8d8a3 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Codecs.kt @@ -18,6 +18,7 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue @@ -321,6 +322,9 @@ object ParsedMessageCodec : AbstractCodec(30u) { @JvmField val MAPPER: ObjectMapper = jacksonObjectMapper().registerModule(JavaTimeModule()) + // otherwise, type supported by JavaTimeModule will be serialized as array of date component + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + // this is required to serialize nulls and empty collections .setSerializationInclusion(JsonInclude.Include.ALWAYS) } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt index f76694250..227ae3f38 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt @@ -19,8 +19,13 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import io.netty.buffer.ByteBufUtil import io.netty.buffer.Unpooled import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.DynamicTest import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestFactory import java.time.Instant +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime class CodecsTest { @@ -121,4 +126,39 @@ class CodecsTest { "unexpected raw body", ) } + + @TestFactory + fun dateTypesTests(): Collection { + val testData = listOf( + LocalDate.now(), + LocalTime.now(), + LocalDateTime.now(), + Instant.now(), + ) + return testData.map { + DynamicTest.dynamicTest("serializes ${it::class.simpleName} as field") { + val parsedMessage = ParsedMessage.builder().apply { + setId(MessageId.DEFAULT) + setType("test") + setBody( + linkedMapOf( + "field" to it, + ) + ) + }.build() + + val dest = Unpooled.buffer() + ParsedMessageCodec.encode(parsedMessage, dest) + val decoded = ParsedMessageCodec.decode(dest) + assertEquals(0, dest.readableBytes()) { "unexpected bytes left: ${ByteBufUtil.hexDump(dest)}" } + + assertEquals(parsedMessage, decoded, "unexpected parsed result decoded") + assertEquals( + "{\"field\":\"$it\"}", + decoded.rawBody.toString(Charsets.UTF_8), + "unexpected raw body", + ) + } + } + } } \ No newline at end of file From c4ab2e1bf993036909bf7166f69f8d154d073ac4 Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Thu, 21 Sep 2023 11:18:12 +0400 Subject: [PATCH 151/154] =?UTF-8?q?[TH2-5069]=20Provided=20the=20ability?= =?UTF-8?q?=20to=20define=20configs=20directory=20using=20the=E2=80=A6=20(?= =?UTF-8?q?#277)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 16 +- build.gradle | 3 + gradle.properties | 2 +- .../schema/factory/AbstractCommonFactory.java | 73 +------- .../common/schema/factory/CommonFactory.java | 175 +++++++----------- .../common/schema/util/Log4jConfigUtils.kt | 4 +- .../configuration/ConfigurationManager.kt | 3 + .../schema/factory/CommonFactoryTest.kt | 123 ++++++++++++ .../impl/rabbitmq/transport/CodecsTest.kt | 23 ++- .../custom.json | 1 + 10 files changed, 224 insertions(+), 199 deletions(-) create mode 100644 src/test/kotlin/com/exactpro/th2/common/schema/factory/CommonFactoryTest.kt create mode 100644 src/test/resources/test_common_factory_load_configs/custom.json diff --git a/README.md b/README.md index 8929e98a9..07f0a973e 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,10 @@ import com.exactpro.th2.common.schema.factory.CommonFactory Then you will create an instance of imported class, by choosing one of the following options: -1. Create factory with configs from the default path (`/var/th2/config/*`): +1. Create factory with configs from the `th2.common.configuration-directory` environment variable or default path (`/var/th2/config/*`): ``` var factory = CommonFactory(); ``` -1. Create factory with configs from the specified file paths: - ``` - var factory = CommonFactory(rabbitMQ, routerMQ, routerGRPC, cradle, custom, prometheus, dictionariesDir); - ``` 1. Create factory with configs from the specified arguments: ``` var factory = CommonFactory.createFromArguments(args); @@ -34,7 +30,8 @@ Then you will create an instance of imported class, by choosing one of the follo * --dictionariesDir - path to the directory which contains files with the encoded dictionaries * --prometheusConfiguration - path to json file with configuration for prometheus metrics server * --boxConfiguration - path to json file with boxes configuration and information - * -c/--configs - folder with json files for schemas configurations with special names: + * -c/--configs - folder with json files for schemas configurations with special names. + If you doesn't specify -c/--configs common factory uses configs from the `th2.common.configuration-directory` environment variable or default path (`/var/th2/config/*`) 1. rabbitMq.json - configuration for RabbitMQ 2. mq.json - configuration for MessageRouter @@ -494,6 +491,11 @@ dependencies { ## Release notes +### 5.5.0-dev + +#### Changed: ++ Provided the ability to define configs directory using the `th2.common.configuration-directory` environment variable + ### 5.4.2-dev #### Fix @@ -919,7 +921,7 @@ dependencies { ### 3.13.0 + reads dictionaries from the /var/th2/config/dictionary folder. -+ uses mq_router, grpc_router, cradle_manager optional JSON configs from the /var/th2/config folder ++ uses mq_router, grpc_router, cradle_manager optional JSON configs from the `/var/th2/config` folder ### 3.11.0 diff --git a/build.gradle b/build.gradle index 78630087b..802991382 100644 --- a/build.gradle +++ b/build.gradle @@ -258,6 +258,9 @@ dependencies { testImplementation 'org.jetbrains.kotlin:kotlin-test-junit5' testImplementation "org.testcontainers:testcontainers:1.17.1" testImplementation "org.testcontainers:rabbitmq:1.17.1" + testImplementation("org.junit-pioneer:junit-pioneer:2.1.0") { + because("system property tests") + } testFixturesImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit5', version: kotlin_version testFixturesImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" diff --git a/gradle.properties b/gradle.properties index b876306b8..2eae5e594 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.4.2 +release_version=5.5.0 description='th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j kapt.include.compile.classpath=false diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index d316c5129..4cc7e554f 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -104,8 +104,8 @@ public abstract class AbstractCommonFactory implements AutoCloseable { * @deprecated please use {@link #LOG4J_PROPERTIES_DEFAULT_PATH} */ @Deprecated - protected static final String LOG4J_PROPERTIES_DEFAULT_PATH_OLD = "/home/etc"; - protected static final String LOG4J_PROPERTIES_DEFAULT_PATH = "/var/th2/config"; + protected static final Path LOG4J_PROPERTIES_DEFAULT_PATH_OLD = Path.of("/home/etc"); + protected static final Path LOG4J_PROPERTIES_DEFAULT_PATH = Path.of("/var/th2/config"); protected static final String LOG4J2_PROPERTIES_NAME = "log4j2.properties"; public static final ObjectMapper MAPPER = new ObjectMapper(); @@ -168,71 +168,6 @@ public AbstractCommonFactory(FactorySettings settings) { stringSubstitutor = new StringSubstitutor(key -> defaultIfBlank(settings.getVariables().get(key), System.getenv(key))); } - /** - * Create factory with default implementation schema classes - * - * @deprecated Please use {@link AbstractCommonFactory#AbstractCommonFactory(FactorySettings)} - */ - @Deprecated(since = "4.0.0", forRemoval = true) - public AbstractCommonFactory() { - this(new FactorySettings()); - } - - /** - * Create factory with non-default implementations schema classes - * - * @param messageRouterParsedBatchClass Class for {@link MessageRouter} which work with {@link MessageBatch} - * @param messageRouterRawBatchClass Class for {@link MessageRouter} which work with {@link RawMessageBatch} - * @param eventBatchRouterClass Class for {@link MessageRouter} which work with {@link EventBatch} - * @param grpcRouterClass Class for {@link GrpcRouter} - * @deprecated Please use {@link AbstractCommonFactory#AbstractCommonFactory(FactorySettings)} - */ - @Deprecated(since = "4.0.0", forRemoval = true) - public AbstractCommonFactory( - @NotNull Class> messageRouterParsedBatchClass, - @NotNull Class> messageRouterRawBatchClass, - @NotNull Class> messageRouterMessageGroupBatchClass, - @NotNull Class> eventBatchRouterClass, - @NotNull Class grpcRouterClass - ) { - this(new FactorySettings() - .messageRouterParsedBatchClass(messageRouterParsedBatchClass) - .messageRouterRawBatchClass(messageRouterRawBatchClass) - .messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass) - .eventBatchRouterClass(eventBatchRouterClass) - .grpcRouterClass(grpcRouterClass) - ); - } - - /** - * Create factory with non-default implementations schema classes - * - * @param messageRouterParsedBatchClass Class for {@link MessageRouter} which work with {@link MessageBatch} - * @param messageRouterRawBatchClass Class for {@link MessageRouter} which work with {@link RawMessageBatch} - * @param eventBatchRouterClass Class for {@link MessageRouter} which work with {@link EventBatch} - * @param grpcRouterClass Class for {@link GrpcRouter} - * @param environmentVariables map with additional environment variables - * @deprecated Please use {@link AbstractCommonFactory#AbstractCommonFactory(FactorySettings)} - */ - @Deprecated(since = "4.0.0", forRemoval = true) - protected AbstractCommonFactory( - @NotNull Class> messageRouterParsedBatchClass, - @NotNull Class> messageRouterRawBatchClass, - @NotNull Class> messageRouterMessageGroupBatchClass, - @NotNull Class> eventBatchRouterClass, - @NotNull Class grpcRouterClass, - @NotNull Map environmentVariables - ) { - this(new FactorySettings() - .messageRouterParsedBatchClass(messageRouterParsedBatchClass) - .messageRouterRawBatchClass(messageRouterRawBatchClass) - .messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass) - .eventBatchRouterClass(eventBatchRouterClass) - .grpcRouterClass(grpcRouterClass) - .variables(environmentVariables) - ); - } - public void start() { DefaultExports.initialize(); PrometheusConfiguration prometheusConfiguration = loadPrometheusConfiguration(); @@ -855,8 +790,8 @@ public void close() { LOGGER.info("Common factory has been closed"); } - protected static void configureLogger(String... paths) { - List listPath = new ArrayList<>(); + protected static void configureLogger(Path... paths) { + List listPath = new ArrayList<>(); listPath.add(LOG4J_PROPERTIES_DEFAULT_PATH); listPath.add(LOG4J_PROPERTIES_DEFAULT_PATH_OLD); listPath.addAll(Arrays.asList(requireNonNull(paths, "Paths can't be null"))); diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index 4374acfbf..8fe4353c3 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -16,28 +16,19 @@ package com.exactpro.th2.common.schema.factory; import com.exactpro.cradle.cassandra.CassandraStorageSettings; -import com.exactpro.th2.common.grpc.EventBatch; -import com.exactpro.th2.common.grpc.MessageBatch; -import com.exactpro.th2.common.grpc.MessageGroupBatch; -import com.exactpro.th2.common.grpc.RawMessageBatch; import com.exactpro.th2.common.metrics.PrometheusConfiguration; import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration; import com.exactpro.th2.common.schema.configuration.ConfigurationManager; import com.exactpro.th2.common.schema.cradle.CradleConfidentialConfiguration; import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration; import com.exactpro.th2.common.schema.dictionary.DictionaryType; -import com.exactpro.th2.common.schema.event.EventBatchRouter; import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration; import com.exactpro.th2.common.schema.grpc.configuration.GrpcRouterConfiguration; import com.exactpro.th2.common.schema.grpc.router.GrpcRouter; -import com.exactpro.th2.common.schema.grpc.router.impl.DefaultGrpcRouter; import com.exactpro.th2.common.schema.message.MessageRouter; import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration; import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration; import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.group.RabbitMessageGroupBatchRouter; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.parsed.RabbitParsedBatchRouter; -import com.exactpro.th2.common.schema.message.impl.rabbitmq.raw.RabbitRawBatchRouter; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.ConfigMapList; import io.fabric8.kubernetes.api.model.Secret; @@ -66,6 +57,7 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Base64; import java.util.HashMap; @@ -79,7 +71,7 @@ import static com.exactpro.th2.common.schema.util.ArchiveUtils.getGzipBase64StringDecoder; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; -import static java.util.Objects.requireNonNullElse; +import static java.util.Objects.requireNonNullElseGet; import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; /** @@ -87,23 +79,25 @@ */ public class CommonFactory extends AbstractCommonFactory { - private static final Path CONFIG_DEFAULT_PATH = Path.of("/var/th2/config/"); - - private static final String RABBIT_MQ_FILE_NAME = "rabbitMQ.json"; - private static final String ROUTER_MQ_FILE_NAME = "mq.json"; - private static final String GRPC_FILE_NAME = "grpc.json"; - private static final String ROUTER_GRPC_FILE_NAME = "grpc_router.json"; - private static final String CRADLE_CONFIDENTIAL_FILE_NAME = "cradle.json"; - private static final String PROMETHEUS_FILE_NAME = "prometheus.json"; - private static final String CUSTOM_FILE_NAME = "custom.json"; - private static final String BOX_FILE_NAME = "box.json"; - private static final String CONNECTION_MANAGER_CONF_FILE_NAME = "mq_router.json"; - private static final String CRADLE_NON_CONFIDENTIAL_FILE_NAME = "cradle_manager.json"; + public static final String TH2_COMMON_SYSTEM_PROPERTY = "th2.common"; + public static final String TH2_COMMON_CONFIGURATION_DIRECTORY_SYSTEM_PROPERTY = TH2_COMMON_SYSTEM_PROPERTY + '.' + "configuration-directory"; + static final Path CONFIG_DEFAULT_PATH = Path.of("/var/th2/config/"); + + static final String RABBIT_MQ_FILE_NAME = "rabbitMQ.json"; + static final String ROUTER_MQ_FILE_NAME = "mq.json"; + static final String GRPC_FILE_NAME = "grpc.json"; + static final String ROUTER_GRPC_FILE_NAME = "grpc_router.json"; + static final String CRADLE_CONFIDENTIAL_FILE_NAME = "cradle.json"; + static final String PROMETHEUS_FILE_NAME = "prometheus.json"; + static final String CUSTOM_FILE_NAME = "custom.json"; + static final String BOX_FILE_NAME = "box.json"; + static final String CONNECTION_MANAGER_CONF_FILE_NAME = "mq_router.json"; + static final String CRADLE_NON_CONFIDENTIAL_FILE_NAME = "cradle_manager.json"; /** @deprecated please use {@link #DICTIONARY_ALIAS_DIR_NAME} */ @Deprecated - private static final String DICTIONARY_TYPE_DIR_NAME = "dictionary"; - private static final String DICTIONARY_ALIAS_DIR_NAME = "dictionaries"; + static final String DICTIONARY_TYPE_DIR_NAME = "dictionary"; + static final String DICTIONARY_ALIAS_DIR_NAME = "dictionaries"; private static final String RABBITMQ_SECRET_NAME = "rabbitmq"; private static final String CASSANDRA_SECRET_NAME = "cassandra"; @@ -123,96 +117,20 @@ public class CommonFactory extends AbstractCommonFactory { private final Path dictionaryTypesDir; private final Path dictionaryAliasesDir; private final Path oldDictionariesDir; - private final ConfigurationManager configurationManager; + final ConfigurationManager configurationManager; private static final Logger LOGGER = LoggerFactory.getLogger(CommonFactory.class.getName()); - /** - * @deprecated Please use {@link CommonFactory#CommonFactory(FactorySettings)} - */ - @Deprecated(since = "4.0.0", forRemoval = true) - protected CommonFactory(Class> messageRouterParsedBatchClass, - Class> messageRouterRawBatchClass, - Class> messageRouterMessageGroupBatchClass, - Class> eventBatchRouterClass, - Class grpcRouterClass, - @Nullable Path custom, - @Nullable Path dictionaryTypesDir, - @Nullable Path dictionaryAliasesDir, - @Nullable Path oldDictionariesDir, - Map environmentVariables) { - this(new FactorySettings() - .messageRouterParsedBatchClass(messageRouterParsedBatchClass) - .messageRouterRawBatchClass(messageRouterRawBatchClass) - .messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass) - .eventBatchRouterClass(eventBatchRouterClass) - .grpcRouterClass(grpcRouterClass) - .variables(environmentVariables) - .custom(custom) - .dictionaryTypesDir(dictionaryTypesDir) - .dictionaryAliasesDir(dictionaryAliasesDir) - .oldDictionariesDir(oldDictionariesDir) - ); - } - public CommonFactory(FactorySettings settings) { super(settings); custom = defaultPathIfNull(settings.getCustom(), CUSTOM_FILE_NAME); dictionaryTypesDir = defaultPathIfNull(settings.getDictionaryTypesDir(), DICTIONARY_TYPE_DIR_NAME); dictionaryAliasesDir = defaultPathIfNull(settings.getDictionaryAliasesDir(), DICTIONARY_ALIAS_DIR_NAME); - oldDictionariesDir = requireNonNullElse(settings.getOldDictionariesDir(), CONFIG_DEFAULT_PATH); + oldDictionariesDir = requireNonNullElseGet(settings.getOldDictionariesDir(), CommonFactory::getConfigPath); configurationManager = createConfigurationManager(settings); start(); } - /** - * @deprecated Please use {@link CommonFactory#CommonFactory(FactorySettings)} - */ - @Deprecated(since = "3.10.0", forRemoval = true) - public CommonFactory(Class> messageRouterParsedBatchClass, - Class> messageRouterRawBatchClass, - Class> messageRouterMessageGroupBatchClass, - Class> eventBatchRouterClass, - Class grpcRouterClass, - Path rabbitMQ, Path routerMQ, Path routerGRPC, Path cradle, Path custom, Path prometheus, Path dictionariesDir, Path boxConfiguration) { - this(new FactorySettings() - .messageRouterParsedBatchClass(messageRouterParsedBatchClass) - .messageRouterRawBatchClass(messageRouterRawBatchClass) - .messageRouterMessageGroupBatchClass(messageRouterMessageGroupBatchClass) - .eventBatchRouterClass(eventBatchRouterClass) - .grpcRouterClass(grpcRouterClass) - .rabbitMQ(rabbitMQ) - .routerMQ(routerMQ) - .routerGRPC(routerGRPC) - .cradleConfidential(cradle) - .prometheus(prometheus) - .boxConfiguration(boxConfiguration) - .custom(custom) - .dictionaryTypesDir(dictionariesDir) - ); - } - - /** - * @deprecated Please use {@link CommonFactory#CommonFactory(FactorySettings)} - */ - @Deprecated(since = "3.10.0", forRemoval = true) - public CommonFactory(Path rabbitMQ, Path routerMQ, Path routerGRPC, Path cradle, Path custom, Path prometheus, Path dictionariesDir, Path boxConfiguration) { - this(RabbitParsedBatchRouter.class, RabbitRawBatchRouter.class, RabbitMessageGroupBatchRouter.class, EventBatchRouter.class, DefaultGrpcRouter.class, - rabbitMQ ,routerMQ ,routerGRPC ,cradle ,custom ,dictionariesDir ,prometheus ,boxConfiguration); - } - - /** - * @deprecated Please use {@link CommonFactory#CommonFactory(FactorySettings)} - */ - @Deprecated(since = "3.10.0", forRemoval = true) - public CommonFactory(Class> messageRouterParsedBatchClass, - Class> messageRouterRawBatchClass, - Class> messageRouterMessageGroupBatchClass, - Class> eventBatchRouterClass, - Class grpcRouterClass) { - this(new FactorySettings(messageRouterParsedBatchClass, messageRouterRawBatchClass, messageRouterMessageGroupBatchClass, eventBatchRouterClass, grpcRouterClass)); - } - public CommonFactory() { this(new FactorySettings()); } @@ -318,7 +236,7 @@ public static CommonFactory createFromArguments(String... args) { try { CommandLine cmd = new DefaultParser().parse(options, args); - String configs = cmd.getOptionValue(configOption.getLongOpt()); + Path configs = getConfigPath(cmd.getOptionValue(configOption.getLongOpt())); if (cmd.hasOption(namespaceOption.getLongOpt()) && cmd.hasOption(boxNameOption.getLongOpt())) { String namespace = cmd.getOptionValue(namespaceOption.getLongOpt()); @@ -349,7 +267,7 @@ public static CommonFactory createFromArguments(String... args) { return createFromKubernetes(namespace, boxName, contextName, dictionaries); } - if (configs != null) { + if (!CONFIG_DEFAULT_PATH.equals(configs)) { configureLogger(configs); } FactorySettings settings = new FactorySettings(); @@ -366,7 +284,7 @@ public static CommonFactory createFromArguments(String... args) { settings.setDictionaryTypesDir(calculatePath(cmd, dictionariesDirOption, configs, DICTIONARY_TYPE_DIR_NAME)); settings.setDictionaryAliasesDir(calculatePath(cmd, dictionariesDirOption, configs, DICTIONARY_ALIAS_DIR_NAME)); String oldDictionariesDir = cmd.getOptionValue(dictionariesDirOption.getLongOpt()); - settings.setOldDictionariesDir(oldDictionariesDir == null ? (configs == null ? CONFIG_DEFAULT_PATH : Path.of(configs)) : Path.of(oldDictionariesDir)); + settings.setOldDictionariesDir(oldDictionariesDir == null ? configs : Path.of(oldDictionariesDir)); return new CommonFactory(settings); } catch (ParseException e) { @@ -477,7 +395,7 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam if (loggingData != null) { writeFile(configPath.resolve(LOG4J2_PROPERTIES_NAME), loggingData); - configureLogger(configPath.toString()); + configureLogger(configPath); } settings.setRabbitMQ(writeFile(configPath, RABBIT_MQ_FILE_NAME, rabbitMqData)); @@ -632,6 +550,41 @@ public InputStream readDictionary(DictionaryType dictionaryType) { } } + static @NotNull Path getConfigPath() { + String pathString = System.getProperty(TH2_COMMON_CONFIGURATION_DIRECTORY_SYSTEM_PROPERTY); + if (pathString != null) { + Path path = Paths.get(pathString); + if (Files.exists(path) && Files.isDirectory(path)) { + return path; + } + LOGGER.warn("'{}' config directory passed via '{}' system property doesn't exist or it is not a directory", + pathString, + TH2_COMMON_CONFIGURATION_DIRECTORY_SYSTEM_PROPERTY); + } else { + LOGGER.debug("Skipped blank environment variable path for configs directory"); + } + + if (!Files.exists(CONFIG_DEFAULT_PATH)) { + LOGGER.error("'{}' default config directory doesn't exist", CONFIG_DEFAULT_PATH); + } + return CONFIG_DEFAULT_PATH; + } + + static @NotNull Path getConfigPath(@Nullable String cmdPath) { + String pathString = StringUtils.trim(cmdPath); + if (pathString != null) { + Path path = Paths.get(pathString); + if (Files.exists(path) && Files.isDirectory(path)) { + return path; + } + LOGGER.warn("'{}' config directory passed via CMD doesn't exist or it is not a directory", cmdPath); + } else { + LOGGER.debug("Skipped blank CMD path for configs directory"); + } + + return getConfigPath(); + } + private static Path writeFile(Path configPath, String fileName, Map configMap) throws IOException { Path file = configPath.resolve(fileName); writeFile(file, configMap.get(fileName)); @@ -712,7 +665,7 @@ private static ConfigurationManager createConfigurationManager(FactorySettings s } private static Path defaultPathIfNull(Path path, String name) { - return path == null ? CONFIG_DEFAULT_PATH.resolve(name) : path; + return path == null ? getConfigPath().resolve(name) : path; } private static void writeFile(Path path, String data) throws IOException { @@ -734,15 +687,15 @@ private static Option createLongOption(Options options, String optionName) { return option; } - private static Path calculatePath(String path, String configsPath, String fileName) { - return path != null ? Path.of(path) : (configsPath != null ? Path.of(configsPath, fileName) : CONFIG_DEFAULT_PATH.resolve(fileName)); + private static Path calculatePath(String path, @NotNull Path configsPath, String fileName) { + return path != null ? Path.of(path) : configsPath.resolve(fileName); } - private static Path calculatePath(CommandLine cmd, Option option, String configs, String fileName) { + private static Path calculatePath(CommandLine cmd, Option option, @NotNull Path configs, String fileName) { return calculatePath(cmd.getOptionValue(option.getLongOpt()), configs, fileName); } - private static Path calculatePath(CommandLine cmd, Option current, Option deprecated, String configs, String fileName) { + private static Path calculatePath(CommandLine cmd, Option current, Option deprecated, @NotNull Path configs, String fileName) { return calculatePath(defaultIfNull(cmd.getOptionValue(current.getLongOpt()), cmd.getOptionValue(deprecated.getLongOpt())), configs, fileName); } } diff --git a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt index b8ca23b45..7562fa357 100644 --- a/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt +++ b/src/main/java/com/exactpro/th2/common/schema/util/Log4jConfigUtils.kt @@ -24,11 +24,11 @@ import org.slf4j.LoggerFactory class Log4jConfigUtils { fun configure( - pathList: List, + pathList: List, fileName: String, ) { pathList.asSequence() - .map { Path.of(it, fileName) } + .map { it.resolve(fileName) } .filter(Files::exists) .firstOrNull() ?.let { path -> diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/configuration/ConfigurationManager.kt b/src/main/kotlin/com/exactpro/th2/common/schema/configuration/ConfigurationManager.kt index cebcad3db..f7eb10d60 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/configuration/ConfigurationManager.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/configuration/ConfigurationManager.kt @@ -26,6 +26,8 @@ import java.util.concurrent.ConcurrentHashMap class ConfigurationManager(private val configurationPath: Map, Path>) { private val configurations: MutableMap, Any?> = ConcurrentHashMap() + operator fun get(clazz: Class<*>): Path? = configurationPath[clazz] + fun loadConfiguration( objectMapper: ObjectMapper, stringSubstitutor: StringSubstitutor, @@ -48,6 +50,7 @@ class ConfigurationManager(private val configurationPath: Map, Path>) { } } + @Suppress("UNCHECKED_CAST") fun getConfigurationOrLoad( objectMapper: ObjectMapper, stringSubstitutor: StringSubstitutor, diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/factory/CommonFactoryTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/factory/CommonFactoryTest.kt new file mode 100644 index 000000000..c918f8879 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/common/schema/factory/CommonFactoryTest.kt @@ -0,0 +1,123 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.common.schema.factory + +import com.exactpro.cradle.cassandra.CassandraStorageSettings +import com.exactpro.th2.common.metrics.PrometheusConfiguration +import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration +import com.exactpro.th2.common.schema.cradle.CradleConfidentialConfiguration +import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration +import com.exactpro.th2.common.schema.factory.CommonFactory.CONFIG_DEFAULT_PATH +import com.exactpro.th2.common.schema.factory.CommonFactory.CUSTOM_FILE_NAME +import com.exactpro.th2.common.schema.factory.CommonFactory.DICTIONARY_ALIAS_DIR_NAME +import com.exactpro.th2.common.schema.factory.CommonFactory.DICTIONARY_TYPE_DIR_NAME +import com.exactpro.th2.common.schema.factory.CommonFactory.TH2_COMMON_CONFIGURATION_DIRECTORY_SYSTEM_PROPERTY +import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration +import com.exactpro.th2.common.schema.grpc.configuration.GrpcRouterConfiguration +import com.exactpro.th2.common.schema.message.configuration.MessageRouterConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.ConnectionManagerConfiguration +import com.exactpro.th2.common.schema.message.impl.rabbitmq.configuration.RabbitMQConfiguration +import org.junit.jupiter.api.Test +import org.junitpioneer.jupiter.SetSystemProperty +import java.nio.file.Path +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + + +class CommonFactoryTest { + + @Test + fun `test load config by default path (default constructor)`() { + CommonFactory().use { commonFactory -> + assertConfigs(commonFactory, CONFIG_DEFAULT_PATH) + } + } + + @Test + fun `test load config by default path (createFromArguments(empty))`() { + CommonFactory.createFromArguments().use { commonFactory -> + assertConfigs(commonFactory, CONFIG_DEFAULT_PATH) + } + } + + @Test + fun `test load config by custom path (createFromArguments(not empty))`() { + CommonFactory.createFromArguments("-c", CONFIG_DIR_IN_RESOURCE).use { commonFactory -> + assertConfigs(commonFactory, Path.of(CONFIG_DIR_IN_RESOURCE)) + } + } + + @Test + @SetSystemProperty(key = TH2_COMMON_CONFIGURATION_DIRECTORY_SYSTEM_PROPERTY, value = CONFIG_DIR_IN_RESOURCE) + fun `test load config by environment variable path (default constructor)`() { + CommonFactory().use { commonFactory -> + assertConfigs(commonFactory, Path.of(CONFIG_DIR_IN_RESOURCE)) + } + } + + @Test + @SetSystemProperty(key = TH2_COMMON_CONFIGURATION_DIRECTORY_SYSTEM_PROPERTY, value = CONFIG_DIR_IN_RESOURCE) + fun `test load config by environment variable path (createFromArguments(empty))`() { + CommonFactory.createFromArguments().use { commonFactory -> + assertConfigs(commonFactory, Path.of(CONFIG_DIR_IN_RESOURCE)) + } + } + + @Test + @SetSystemProperty(key = TH2_COMMON_CONFIGURATION_DIRECTORY_SYSTEM_PROPERTY, value = CONFIG_DIR_IN_RESOURCE) + fun `test load config by custom path (createFromArguments(not empty) + environment variable)`() { + CommonFactory.createFromArguments("-c", CONFIG_DIR_IN_RESOURCE).use { commonFactory -> + assertConfigs(commonFactory, Path.of(CONFIG_DIR_IN_RESOURCE)) + } + } + + + private fun assertConfigs(commonFactory: CommonFactory, configPath: Path) { + CONFIG_NAME_TO_COMMON_FACTORY_SUPPLIER.forEach { (configName, actualPathSupplier) -> + assertEquals(configPath.resolve(configName), commonFactory.actualPathSupplier(), "Configured config path: $configPath, config name: $configName") + } + assertConfigurationManager(commonFactory, configPath) + } + + private fun assertConfigurationManager(commonFactory: CommonFactory, configPath: Path) { + CONFIG_CLASSES.forEach { clazz -> + assertNotNull(commonFactory.configurationManager[clazz]) + assertEquals(configPath, commonFactory.configurationManager[clazz]?.parent , "Configured config path: $configPath, config class: $clazz") + } + } + + companion object { + private const val CONFIG_DIR_IN_RESOURCE = "src/test/resources/test_common_factory_load_configs" + + private val CONFIG_NAME_TO_COMMON_FACTORY_SUPPLIER: Map Path> = mapOf( + CUSTOM_FILE_NAME to { pathToCustomConfiguration }, + DICTIONARY_ALIAS_DIR_NAME to { pathToDictionaryAliasesDir }, + DICTIONARY_TYPE_DIR_NAME to { pathToDictionaryTypesDir }, + ) + + private val CONFIG_CLASSES: Set> = setOf( + RabbitMQConfiguration::class.java, + MessageRouterConfiguration::class.java, + ConnectionManagerConfiguration::class.java, + GrpcConfiguration::class.java, + GrpcRouterConfiguration::class.java, + CradleConfidentialConfiguration::class.java, + CradleNonConfidentialConfiguration::class.java, + CassandraStorageSettings::class.java, + PrometheusConfiguration::class.java, + BoxConfiguration::class.java, + ) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt index 227ae3f38..c4c706bdc 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/CodecsTest.kt @@ -26,6 +26,8 @@ import java.time.Instant import java.time.LocalDate import java.time.LocalDateTime import java.time.LocalTime +import java.time.format.DateTimeFormatter +import java.time.temporal.TemporalAccessor class CodecsTest { @@ -129,20 +131,23 @@ class CodecsTest { @TestFactory fun dateTypesTests(): Collection { - val testData = listOf( - LocalDate.now(), - LocalTime.now(), - LocalDateTime.now(), - Instant.now(), + LocalTime.parse("16:36:38.035420").toString() + val testData = listOf String>>( + LocalDate.now() to TemporalAccessor::toString, + LocalTime.now() to DateTimeFormatter.ISO_LOCAL_TIME::format, + // Check case when LocalTime.toString() around nanos to 1000 + LocalTime.parse("16:36:38.035420") to DateTimeFormatter.ISO_LOCAL_TIME::format, + LocalDateTime.now() to DateTimeFormatter.ISO_LOCAL_DATE_TIME::format, + Instant.now() to TemporalAccessor::toString, ) - return testData.map { - DynamicTest.dynamicTest("serializes ${it::class.simpleName} as field") { + return testData.map { (value, formatter) -> + DynamicTest.dynamicTest("serializes ${value::class.simpleName} as field") { val parsedMessage = ParsedMessage.builder().apply { setId(MessageId.DEFAULT) setType("test") setBody( linkedMapOf( - "field" to it, + "field" to value, ) ) }.build() @@ -154,7 +159,7 @@ class CodecsTest { assertEquals(parsedMessage, decoded, "unexpected parsed result decoded") assertEquals( - "{\"field\":\"$it\"}", + "{\"field\":\"${formatter(value)}\"}", decoded.rawBody.toString(Charsets.UTF_8), "unexpected raw body", ) diff --git a/src/test/resources/test_common_factory_load_configs/custom.json b/src/test/resources/test_common_factory_load_configs/custom.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/test/resources/test_common_factory_load_configs/custom.json @@ -0,0 +1 @@ +{} \ No newline at end of file From 0bb498423714098d590ef8d74d7795f2b211d71c Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Fri, 13 Oct 2023 17:20:18 +0400 Subject: [PATCH 152/154] Add methods to check if the field is set in builders (#279) * Add methods to check if the field is set * Update version and readme * Correct error messages. Add check for sequence * Change property to method for checking type * Add methods for Java binary compatibility --- README.md | 7 +- gradle.properties | 2 +- .../impl/rabbitmq/transport/EventId.kt | 92 ++++++++++++- .../impl/rabbitmq/transport/Message.kt | 1 + .../impl/rabbitmq/transport/MessageId.kt | 121 +++++++++++++++++- .../impl/rabbitmq/transport/ParsedMessage.kt | 17 ++- .../impl/rabbitmq/transport/RawMessage.kt | 118 ++++++++++++++++- 7 files changed, 339 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 07f0a973e..eab564fd5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2 common library (Java) (5.4.2) +# th2 common library (Java) (5.6.0) ## Usage @@ -491,6 +491,11 @@ dependencies { ## Release notes +### 5.6.0-dev + +#### Added: ++ New methods for transport message builders which allows checking whether the field is set or not + ### 5.5.0-dev #### Changed: diff --git a/gradle.properties b/gradle.properties index 2eae5e594..5962a2f74 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -release_version=5.5.0 +release_version=5.6.0 description='th2 common library (Java)' vcs_url=https://github.com/th2-net/th2-common-j kapt.include.compile.classpath=false diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt index 152227d45..98f140755 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/EventId.kt @@ -16,8 +16,8 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import com.google.auto.value.AutoBuilder import java.time.Instant +import java.util.StringJoiner data class EventId( val id: String, @@ -50,12 +50,15 @@ data class EventId( return "EventId(id='$id', book='$book', scope='$scope', timestamp=$timestamp)" } - @AutoBuilder interface Builder { val id: String + fun isIdSet(): Boolean val book: String + fun isBookSet(): Boolean val scope: String + fun isScopeSet(): Boolean val timestamp: Instant + fun isTimestampSet(): Boolean fun setId(id: String): Builder fun setBook(book: String): Builder @@ -64,10 +67,89 @@ data class EventId( fun build(): EventId } - fun toBuilder(): Builder = AutoBuilder_EventId_Builder(this) + fun toBuilder(): Builder = BuilderImpl(this) companion object { @JvmStatic - fun builder(): Builder = AutoBuilder_EventId_Builder() + fun builder(): Builder = BuilderImpl() } -} \ No newline at end of file + +} + +private class BuilderImpl : EventId.Builder { + private var _id: String? = null + private var _book: String? = null + private var _scope: String? = null + private var _timestamp: Instant? = null + + constructor() + constructor(source: EventId) { + _id = source.id + _book = source.book + _scope = source.scope + _timestamp = source.timestamp + } + + override fun setId(id: String): EventId.Builder = apply { + this._id = id + } + + override val id: String + get() = checkNotNull(_id) { "Property \"id\" has not been set" } + + override fun isIdSet(): Boolean = _id != null + + override fun setBook(book: String): EventId.Builder = apply { + this._book = book + } + + override val book: String + get() = checkNotNull(_book) { "Property \"book\" has not been set" } + + override fun isBookSet(): Boolean = _book != null + + override fun setScope(scope: String): EventId.Builder = apply { + this._scope = scope + } + + override val scope: String + get() = checkNotNull(_scope) { "Property \"scope\" has not been set" } + + override fun isScopeSet(): Boolean = _scope != null + + override fun setTimestamp(timestamp: Instant): EventId.Builder = apply { + this._timestamp = timestamp + } + + override val timestamp: Instant + get() { + return checkNotNull(_timestamp) { "Property \"timestamp\" has not been set" } + } + + override fun isTimestampSet(): Boolean = _timestamp != null + + override fun build(): EventId { + if (_id == null || _book == null || _scope == null || _timestamp == null) { + val missing = StringJoiner(",", "[", "]") + if (_id == null) { + missing.add("id") + } + if (_book == null) { + missing.add("book") + } + if (_scope == null) { + missing.add("scope") + } + if (_timestamp == null) { + missing.add("timestamp") + } + error("Missing required properties: $missing") + } + return EventId( + _id!!, + _book!!, + _scope!!, + _timestamp!!, + ) + } +} diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt index 69188708e..a109c63f5 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/Message.kt @@ -30,6 +30,7 @@ interface Message { interface Builder> { val protocol: String + fun isProtocolSet(): Boolean val eventId: EventId? fun setId(id: MessageId): T diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt index 777855d20..e15d9f339 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt @@ -17,8 +17,8 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.CollectionBuilder -import com.google.auto.value.AutoBuilder import java.time.Instant +import java.util.StringJoiner data class MessageId( val sessionAlias: String, @@ -55,12 +55,15 @@ data class MessageId( return "MessageId(sessionAlias='$sessionAlias', direction=$direction, sequence=$sequence, timestamp=$timestamp, subsequence=$subsequence)" } - @AutoBuilder interface Builder { val sessionAlias: String + fun isSessionAliasSet(): Boolean val direction: Direction + fun isDirectionSet(): Boolean val sequence: Long + fun isSequenceSet(): Boolean val timestamp: Instant + fun isTimestampSet(): Boolean fun setSessionAlias(sessionAlias: String): Builder fun setDirection(direction: Direction): Builder @@ -75,13 +78,121 @@ data class MessageId( fun build(): MessageId } - fun toBuilder(): Builder = AutoBuilder_MessageId_Builder(this) + fun toBuilder(): Builder = MessageIdBuilderImpl(this) companion object { @JvmStatic val DEFAULT: MessageId = MessageId("", Direction.OUTGOING, 0, Instant.EPOCH) @JvmStatic - fun builder(): Builder = AutoBuilder_MessageId_Builder() + fun builder(): Builder = MessageIdBuilderImpl() } -} \ No newline at end of file +} + + +private const val SEQUENCE_NOT_SET = -1L + +private class MessageIdBuilderImpl : MessageId.Builder { + private var _sessionAlias: String? = null + private var _direction: Direction? = null + private var _sequence: Long = SEQUENCE_NOT_SET + private var _timestamp: Instant? = null + private var _subsequenceBuilder: CollectionBuilder? = null + private var _subsequence: List = emptyList() + + constructor() + constructor(source: MessageId) { + _sessionAlias = source.sessionAlias + _direction = source.direction + _sequence = source.sequence + _timestamp = source.timestamp + _subsequence = source.subsequence + } + + override fun setSessionAlias(sessionAlias: String): MessageId.Builder = apply { + this._sessionAlias = sessionAlias + } + + override val sessionAlias: String + get() = checkNotNull(_sessionAlias) { "Property \"sessionAlias\" has not been set" } + + override fun isSessionAliasSet(): Boolean = _sessionAlias != null + + override fun setDirection(direction: Direction): MessageId.Builder = apply { + this._direction = direction + } + + override val direction: Direction + get() = checkNotNull(_direction) { "Property \"direction\" has not been set" } + + override fun isDirectionSet(): Boolean = _direction != null + + override fun setSequence(sequence: Long): MessageId.Builder = apply { + require(sequence > SEQUENCE_NOT_SET) { "Property \"sequence\" should not be negative" } + this._sequence = sequence + } + + override val sequence: Long + get() { + check(_sequence != SEQUENCE_NOT_SET) { "Property \"sequence\" has not been set" } + return _sequence + } + + override fun isSequenceSet(): Boolean = _sequence != SEQUENCE_NOT_SET + + override fun setTimestamp(timestamp: Instant): MessageId.Builder = apply { + this._timestamp = timestamp + } + + override val timestamp: Instant + get() = checkNotNull(_timestamp) { "Property \"timestamp\" has not been set" } + + override fun isTimestampSet(): Boolean = _timestamp != null + + override fun setSubsequence(subsequence: List): MessageId.Builder = apply { + check(_subsequenceBuilder == null) { "Cannot set subsequence after calling subsequenceBuilder()" } + this._subsequence = subsequence + } + + override fun subsequenceBuilder(): CollectionBuilder { + if (_subsequenceBuilder == null) { + if (_subsequence.isEmpty()) { + _subsequenceBuilder = CollectionBuilder() + } else { + _subsequenceBuilder = CollectionBuilder().apply { + addAll(_subsequence) + } + _subsequence = emptyList() + } + } + return checkNotNull(_subsequenceBuilder) { "subsequenceBuilder" } + } + + override fun build(): MessageId { + _subsequence = _subsequenceBuilder?.build() ?: _subsequence + if (_sessionAlias == null || _direction == null || _sequence == SEQUENCE_NOT_SET || _timestamp == null) { + val missing = StringJoiner(",", "[", "]") + if (_sessionAlias == null) { + missing.add("sessionAlias") + } + if (_direction == null) { + missing.add("direction") + } + if (_sequence == SEQUENCE_NOT_SET) { + missing.add("sequence") + } + if (_timestamp == null) { + missing.add("timestamp") + } + error("Missing required properties: $missing") + } + return MessageId( + _sessionAlias!!, + _direction!!, + _sequence, + _timestamp!!, + _subsequence, + ) + } +} + diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt index 57784b7a3..cf5d9ccbb 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessage.kt @@ -87,6 +87,7 @@ class ParsedMessage private constructor( interface Builder> : Message.Builder { val type: String + fun isTypeSet(): Boolean fun setType(type: String): T override fun build(): ParsedMessage @@ -155,7 +156,16 @@ class ParsedMessage private constructor( companion object { - val EMPTY = ParsedMessage(type = "", body = emptyMap()) + @JvmField + val EMPTY: ParsedMessage = ParsedMessage(type = "", body = emptyMap()) + + @Deprecated( + "Please use EMPTY instead. Added for binary compatibility", + level = DeprecationLevel.HIDDEN, + replaceWith = ReplaceWith(expression = "EMPTY") + ) + @JvmStatic + fun getEMPTY(): ParsedMessage = EMPTY /** * We want to be able to identify the default body by reference. @@ -164,6 +174,7 @@ class ParsedMessage private constructor( */ private val DEFAULT_BODY: Map = Collections.unmodifiableMap(emptyMap()) + @JvmField val DEFAULT_BODY_SUPPLIER: (ByteBuf) -> Map = { emptyMap() } @JvmStatic @@ -187,11 +198,15 @@ private sealed class BaseParsedBuilder> : Parse override val protocol: String get() = this._protocol ?: "" + + override fun isProtocolSet(): Boolean = _protocol != null override val type: String get() = requireNotNull(this._type) { "Property \"type\" has not been set" } + override fun isTypeSet(): Boolean = _type != null + override fun setId(id: MessageId): T = self { require(idBuilder == null) { "cannot set id after calling idBuilder()" diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt index 98de48fb0..0defb4862 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/RawMessage.kt @@ -16,7 +16,8 @@ package com.exactpro.th2.common.schema.message.impl.rabbitmq.transport -import com.google.auto.value.AutoBuilder +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageId.Companion.builder +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.MapBuilder import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBufUtil.hexDump import io.netty.buffer.Unpooled @@ -57,10 +58,10 @@ data class RawMessage( return "RawMessage(id=$id, eventId=$eventId, metadata=$metadata, protocol='$protocol', body=${hexDump(body)})" } - @AutoBuilder interface Builder : Message.Builder { val body: ByteBuf + fun isBodySet(): Boolean fun setBody(body: ByteBuf): Builder fun setBody(data: ByteArray): Builder = setBody(Unpooled.wrappedBuffer(data)) @@ -72,13 +73,118 @@ data class RawMessage( override fun build(): RawMessage } - fun toBuilder(): Builder = AutoBuilder_RawMessage_Builder(this) + fun toBuilder(): Builder = RawMessageBuilderImpl(this) companion object { - val EMPTY = RawMessage() + @JvmField + val EMPTY: RawMessage = RawMessage() + @Deprecated( + "Please use EMPTY instead. Added for binary compatibility", + level = DeprecationLevel.HIDDEN, + replaceWith = ReplaceWith(expression = "EMPTY") + ) @JvmStatic - fun builder(): Builder = AutoBuilder_RawMessage_Builder() + fun getEMPTY(): RawMessage = EMPTY + + @JvmStatic + fun builder(): Builder = RawMessageBuilderImpl() + } +} + + +internal class RawMessageBuilderImpl : RawMessage.Builder { + private var _idBuilder: MessageId.Builder? = null + private var _id: MessageId? = null + private var _eventId: EventId? = null + private var _metadataBuilder: MapBuilder? = null + private var metadata: Map? = null + private var _protocol: String? = null + private var _body: ByteBuf? = null + + constructor() + constructor(source: RawMessage) { + _id = source.id + _eventId = source.eventId + metadata = source.metadata + _protocol = source.protocol + _body = source.body + } + + override fun setId(id: MessageId): RawMessage.Builder = apply { + check(_idBuilder == null) { "Cannot set id after calling idBuilder()" } + this._id = id + } + + override fun idBuilder(): MessageId.Builder { + if (_idBuilder == null) { + _idBuilder = _id?.toBuilder()?.also { + _id = null + } ?: builder() + } + return _idBuilder!! + } + + override fun setEventId(eventId: EventId): RawMessage.Builder = apply { + this._eventId = eventId + return this + } + + override val eventId: EventId? + get() = _eventId + + override fun setMetadata(metadata: Map): RawMessage.Builder = apply { + check(_metadataBuilder == null) { "Cannot set metadata after calling metadataBuilder()" } + this.metadata = metadata } -} \ No newline at end of file + + override fun metadataBuilder(): MapBuilder { + if (_metadataBuilder == null) { + if (metadata == null) { + _metadataBuilder = MapBuilder() + } else { + _metadataBuilder = MapBuilder().apply { + metadata?.let(this::putAll) + } + metadata = null + } + } + return _metadataBuilder!! + } + + override fun setProtocol(protocol: String): RawMessage.Builder = apply { + this._protocol = protocol + } + + override val protocol: String + get() { + return checkNotNull(_protocol) { "Property \"protocol\" has not been set" } + } + + override fun isProtocolSet(): Boolean = _protocol != null + + override fun setBody(body: ByteBuf): RawMessage.Builder = apply { + this._body = body + } + + override val body: ByteBuf + get() = checkNotNull(_body) { "Property \"body\" has not been set" } + + override fun isBodySet(): Boolean = _body != null + + override fun build(): RawMessage { + _id = _idBuilder?.build() + ?: _id + metadata = _metadataBuilder?.build() + ?: metadata + return RawMessage( + _id ?: MessageId.DEFAULT, + _eventId, + metadata ?: emptyMap(), + _protocol ?: "", + _body ?: Unpooled.EMPTY_BUFFER, + ) + } +} + From 333206d29b0cce5858e00604330fbec0d40d94e3 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Fri, 13 Oct 2023 18:03:47 +0400 Subject: [PATCH 153/154] Fix random generated numbers for parsed message (#281) * Fix random generated numbers for parsed message * Use long min value as a marker for unset sequence * Correct range for sequence in test --- .../schema/message/impl/rabbitmq/transport/MessageId.kt | 4 ++-- .../message/impl/rabbitmq/transport/ParsedMessageTest.kt | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt index e15d9f339..5bd43a5d6 100644 --- a/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt +++ b/src/main/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/MessageId.kt @@ -90,7 +90,7 @@ data class MessageId( } -private const val SEQUENCE_NOT_SET = -1L +private const val SEQUENCE_NOT_SET = Long.MIN_VALUE private class MessageIdBuilderImpl : MessageId.Builder { private var _sessionAlias: String? = null @@ -128,7 +128,7 @@ private class MessageIdBuilderImpl : MessageId.Builder { override fun isDirectionSet(): Boolean = _direction != null override fun setSequence(sequence: Long): MessageId.Builder = apply { - require(sequence > SEQUENCE_NOT_SET) { "Property \"sequence\" should not be negative" } + require(sequence != SEQUENCE_NOT_SET) { "Value $sequence for property \"sequence\" is reserved" } this._sequence = sequence } diff --git a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt index 2560cb198..23d092e27 100644 --- a/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt +++ b/src/test/kotlin/com/exactpro/th2/common/schema/message/impl/rabbitmq/transport/ParsedMessageTest.kt @@ -192,7 +192,10 @@ class ParsedMessageTest { private const val TEST_PROTOCOL = "test-protocol" private const val TEST_SESSION_ALIAS = "test-session-alias" private const val TEST_MESSAGE_TYPE = "test-message-type" - private val TEST_SEQUENCE = Random.nextLong() - private val TEST_SUB_SEQUENCE = listOf(Random.nextInt(), Random.nextInt()) + private val TEST_SEQUENCE = Random.nextLong(from = Long.MIN_VALUE + 1, until = Long.MAX_VALUE) + private val TEST_SUB_SEQUENCE = listOf( + Random.nextInt(from = 0, until = Int.MAX_VALUE), + Random.nextInt(from = 0, until = Int.MAX_VALUE), + ) } } \ No newline at end of file From e1592b3c9901d6c32375c044b3d7cd6b63989196 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Fri, 13 Oct 2023 18:04:28 +0400 Subject: [PATCH 154/154] Add register kotlin and time modules for events body serialization (#280) * Add register kotlin and time modules for events body serialization * Update readme --- README.md | 1 + .../com/exactpro/th2/common/event/Event.java | 14 +++++++- .../exactpro/th2/common/event/TestEvent.kt | 32 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eab564fd5..75f21ccc7 100644 --- a/README.md +++ b/README.md @@ -495,6 +495,7 @@ dependencies { #### Added: + New methods for transport message builders which allows checking whether the field is set or not ++ Serialization support for date time types (e.g. Instant, LocalDateTime/Date/Time) to event body serialization ### 5.5.0-dev diff --git a/src/main/java/com/exactpro/th2/common/event/Event.java b/src/main/java/com/exactpro/th2/common/event/Event.java index d4c291c49..0f2e2143a 100644 --- a/src/main/java/com/exactpro/th2/common/event/Event.java +++ b/src/main/java/com/exactpro/th2/common/event/Event.java @@ -22,6 +22,10 @@ import com.exactpro.th2.common.grpc.MessageID; import com.exactpro.th2.common.message.MessageUtils; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.kotlin.KotlinFeature; +import com.fasterxml.jackson.module.kotlin.KotlinModule; import com.google.protobuf.ByteString; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -61,7 +65,15 @@ public class Event { public static final String UNKNOWN_EVENT_NAME = "Unknown event name"; public static final String UNKNOWN_EVENT_TYPE = "Unknown event type"; public static final EventID DEFAULT_EVENT_ID = EventID.getDefaultInstance(); - protected static final ThreadLocal OBJECT_MAPPER = ThreadLocal.withInitial(() -> new ObjectMapper().setSerializationInclusion(NON_NULL)); + protected static final ThreadLocal OBJECT_MAPPER = ThreadLocal.withInitial(() -> + new ObjectMapper() + .registerModule(new KotlinModule.Builder() + .enable(KotlinFeature.SingletonSupport) + .build()) + .registerModule(new JavaTimeModule()) + // otherwise, type supported by JavaTimeModule will be serialized as array of date component + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .setSerializationInclusion(NON_NULL)); protected final String UUID = generateUUID(); protected final AtomicLong ID_COUNTER = new AtomicLong(); diff --git a/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt b/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt index 4e7c40e35..3bcbacc87 100644 --- a/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt +++ b/src/test/kotlin/com/exactpro/th2/common/event/TestEvent.kt @@ -32,6 +32,10 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll import java.time.Instant +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.time.ZoneOffset typealias ProtoEvent = com.exactpro.th2.common.grpc.Event @@ -273,6 +277,34 @@ class TestEvent { checkEventStatus(listOf(batch), 1, 0) } + @Test + fun `serializes date time fields`() { + class TestBody( + val instant: Instant, + val dateTime: LocalDateTime, + val date: LocalDate, + val time: LocalTime, + ) : IBodyData + + // Friday, 13 October 2023 y., 12:35:05 + val instant = Instant.ofEpochSecond(1697200505) + val protoEvent = Event.start().endTimestamp() + .bodyData( + TestBody( + instant = instant, + dateTime = LocalDateTime.ofInstant(instant, ZoneOffset.UTC), + date = LocalDate.ofInstant(instant, ZoneOffset.UTC), + time = LocalTime.ofInstant(instant, ZoneOffset.UTC), + ) + ).toProto(parentEventId) + val jsonBody = protoEvent.body.toStringUtf8() + assertEquals( + """[{"instant":"2023-10-13T12:35:05Z","dateTime":"2023-10-13T12:35:05","date":"2023-10-13","time":"12:35:05"}]""", + jsonBody, + "unexpected JSON body", + ) + } + private fun com.exactpro.th2.common.grpc.Event.checkDefaultEventFields() { assertAll( { assertTrue(hasId()) },