From 6e54dbcf051cfc06e34f371a868d863cc1e61b9c Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Fri, 20 Oct 2023 16:59:27 +0400 Subject: [PATCH] [TH2-5106] disable cradle message check by default (#73) --- README.md | 5 ++++- .../com/exactpro/th2/lwdataprovider/Context.kt | 3 ++- .../lwdataprovider/configuration/Configuration.kt | 2 ++ .../th2/lwdataprovider/db/CradleMessageExtractor.kt | 12 +++++++++--- .../lwdataprovider/db/TestCradleMessageExtractor.kt | 8 +++++--- .../db/integration/AbstractCradleIntegrationTest.kt | 1 + .../handlers/TestSearchMessagesHandler.kt | 2 +- 7 files changed, 24 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 159cfc1d..239d4280 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,7 @@ spec: # maxBufferDecodeQueue: 10000 # buffer size for messages that sent to decode but answers hasn't been received # decodingTimeout: 60000 # timeout expecting answers from codec. # batchSizeBytes: 256KB # the max size of the batch in bytes. You can use 'MB,KB' suffixes or a plain int value +# validateCradleData: false # validate data loaded from cradle. NOTE: Enabled validation affect performance # codecUsePinAttributes: true # send raw message to specified codec (true) or send to all codecs (false) # responseFormats: string list # resolve data for selected formats only. (allowed values: BASE_64, PARSED) @@ -225,7 +226,9 @@ spec: ## 2.4.0 -+ Add `batchSizeBytes` parameter to limit batch size by size in bytes rather than count of messages. ++ Added `batchSizeBytes` parameter to limit batch size by size in bytes rather than count of messages. ++ Added `validateCradleData` parameter to enable/disable validation logic for data loaded from cradle. + Currently, managed validation logic includes check for message sequence and timestamp inside a group batch. + Parameters `batchSize` and `groupRequestBuffer` removed. The maximum batch size in messages is computed based on `bufferPerQuery` or `maxBufferDecodeQueue` if previous parameter is not set. diff --git a/src/main/kotlin/com/exactpro/th2/lwdataprovider/Context.kt b/src/main/kotlin/com/exactpro/th2/lwdataprovider/Context.kt index fe71a3f1..9a0b7487 100644 --- a/src/main/kotlin/com/exactpro/th2/lwdataprovider/Context.kt +++ b/src/main/kotlin/com/exactpro/th2/lwdataprovider/Context.kt @@ -74,7 +74,8 @@ class Context( ), val cradleMsgExtractor: CradleMessageExtractor = CradleMessageExtractor( cradleManager, - DataMeasurementHistogram.create(registry, "cradle message") + DataMeasurementHistogram.create(registry, "cradle message"), + configuration.validateCradleData ), val generalCradleExtractor: GeneralCradleExtractor = GeneralCradleExtractor(cradleManager), val execExecutor: Executor = Executors.newFixedThreadPool( diff --git a/src/main/kotlin/com/exactpro/th2/lwdataprovider/configuration/Configuration.kt b/src/main/kotlin/com/exactpro/th2/lwdataprovider/configuration/Configuration.kt index b4511f40..548a4814 100644 --- a/src/main/kotlin/com/exactpro/th2/lwdataprovider/configuration/Configuration.kt +++ b/src/main/kotlin/com/exactpro/th2/lwdataprovider/configuration/Configuration.kt @@ -41,6 +41,7 @@ class CustomConfigurationClass( val codecUsePinAttributes: Boolean? = null, val listOfMessageAsSingleMessage: Boolean? = null, val useTransportMode: Boolean? = null, + val validateCradleData: Boolean? = null, val flushSseAfter: Int? = null, val gzipCompressionLevel: Int? = null, @JsonDeserialize(using = ByteSizeDeserializer::class) @@ -70,6 +71,7 @@ class Configuration(customConfiguration: CustomConfigurationClass) { val codecUsePinAttributes: Boolean = VariableBuilder.getVariable(customConfiguration::codecUsePinAttributes, true) val listOfMessageAsSingleMessage: Boolean = VariableBuilder.getVariable(customConfiguration::listOfMessageAsSingleMessage, true) val useTransportMode: Boolean = VariableBuilder.getVariable(customConfiguration::useTransportMode, false) + val validateCradleData: Boolean = VariableBuilder.getVariable(customConfiguration::validateCradleData, false) val flushSseAfter: Int = VariableBuilder.getVariable(customConfiguration::flushSseAfter, 0) val gzipCompressionLevel: Int = VariableBuilder.getVariable(customConfiguration::gzipCompressionLevel, -1) val batchSizeBytes: Int = VariableBuilder.getVariable(customConfiguration::batchSizeBytes, 256 * 1024) diff --git a/src/main/kotlin/com/exactpro/th2/lwdataprovider/db/CradleMessageExtractor.kt b/src/main/kotlin/com/exactpro/th2/lwdataprovider/db/CradleMessageExtractor.kt index 76277cc7..10a48ebc 100644 --- a/src/main/kotlin/com/exactpro/th2/lwdataprovider/db/CradleMessageExtractor.kt +++ b/src/main/kotlin/com/exactpro/th2/lwdataprovider/db/CradleMessageExtractor.kt @@ -45,6 +45,7 @@ import kotlin.system.measureTimeMillis class CradleMessageExtractor( cradleManager: CradleManager, private val dataMeasurement: DataMeasurement, + private val validateCradleData: Boolean ) { private val storage: CradleStorage = cradleManager.storage @@ -120,7 +121,7 @@ class CradleMessageExtractor( val orderStrategy = filter.order?.toOrderStrategy() ?: OrderStrategy.DIRECT val iterator: Iterator = measure("init_groups") { storage.getGroupedMessageBatches(filter) } - .withCheck() + .withCheck(validateCradleData) .withMeasurements("groups", dataMeasurement) if (!iterator.hasNext()) { logger.info { "Empty response received from cradle" } @@ -171,7 +172,7 @@ class CradleMessageExtractor( } tryDrain(group, buffer, sink) } else { - orderStrategy.reorder(prev.messages).forEachIndexed { index, msg -> + orderStrategy.reorder(prev.messages).forEachIndexed { _, msg -> if ((needFiltration && !msg.inRange()) || parameters.preFilter?.invoke(msg) == false) { return@forEachIndexed } @@ -358,8 +359,13 @@ internal class GroupBatchCheckIterator( } companion object { - fun Iterator.withCheck(): Iterator = + fun Iterator.withCheck( + validateCradleData: Boolean + ): Iterator = if (validateCradleData) { GroupBatchCheckIterator(this) + } else { + this + } } } diff --git a/src/test/kotlin/com/exactpro/th2/lwdataprovider/db/TestCradleMessageExtractor.kt b/src/test/kotlin/com/exactpro/th2/lwdataprovider/db/TestCradleMessageExtractor.kt index 8be5de8e..4e2facce 100644 --- a/src/test/kotlin/com/exactpro/th2/lwdataprovider/db/TestCradleMessageExtractor.kt +++ b/src/test/kotlin/com/exactpro/th2/lwdataprovider/db/TestCradleMessageExtractor.kt @@ -81,7 +81,7 @@ internal class TestCradleMessageExtractor { internal fun setUp() { storage = mock { } manager = mock { on { this.storage }.thenReturn(storage) } - extractor = CradleMessageExtractor(manager, DummyDataMeasurement) + extractor = CradleMessageExtractor(manager, DummyDataMeasurement, false) clearInvocations(storage, messageRouter, manager) } @@ -305,6 +305,7 @@ internal class TestCradleMessageExtractor { @ParameterizedTest @EnumSource(Order::class) fun getMessagesGroupUnorderedMessagesByTimestamp(order: Order) { + val extractorWithValidation = CradleMessageExtractor(manager, DummyDataMeasurement, true) val correctMessages = listOf( createCradleStoredMessage( TEST_SESSION_ALIAS, @@ -333,7 +334,7 @@ internal class TestCradleMessageExtractor { whenever(storage.getGroupedMessageBatches(any())).thenReturn(ListCradleResult(batchesList)) val exception = assertThrowsExactly(IllegalStateException::class.java) { - extractor.getMessagesGroup( + extractorWithValidation.getMessagesGroup( GroupedMessageFilter.builder() .bookId(BookId("book")) .groupName("test") @@ -360,6 +361,7 @@ internal class TestCradleMessageExtractor { @ParameterizedTest @EnumSource(Order::class) fun getMessagesGroupUnorderedMessagesBySequence(order: Order) { + val extractorWithValidation = CradleMessageExtractor(manager, DummyDataMeasurement, true) val now = Instant.now() val correctMessages = listOf( createCradleStoredMessage( @@ -389,7 +391,7 @@ internal class TestCradleMessageExtractor { whenever(storage.getGroupedMessageBatches(any())).thenReturn(ListCradleResult(batchesList)) val exception = assertThrowsExactly(IllegalStateException::class.java) { - extractor.getMessagesGroup( + extractorWithValidation.getMessagesGroup( GroupedMessageFilter.builder() .bookId(BookId("book")) .groupName("test") diff --git a/src/test/kotlin/com/exactpro/th2/lwdataprovider/db/integration/AbstractCradleIntegrationTest.kt b/src/test/kotlin/com/exactpro/th2/lwdataprovider/db/integration/AbstractCradleIntegrationTest.kt index 577140d0..4ed39f2c 100644 --- a/src/test/kotlin/com/exactpro/th2/lwdataprovider/db/integration/AbstractCradleIntegrationTest.kt +++ b/src/test/kotlin/com/exactpro/th2/lwdataprovider/db/integration/AbstractCradleIntegrationTest.kt @@ -75,6 +75,7 @@ abstract class AbstractCradleIntegrationTest { messageExtractor = CradleMessageExtractor( cradleManager = cradleManager, DummyDataMeasurement, + false ) } diff --git a/src/test/kotlin/com/exactpro/th2/lwdataprovider/handlers/TestSearchMessagesHandler.kt b/src/test/kotlin/com/exactpro/th2/lwdataprovider/handlers/TestSearchMessagesHandler.kt index 11f65ff3..2aa50ba2 100644 --- a/src/test/kotlin/com/exactpro/th2/lwdataprovider/handlers/TestSearchMessagesHandler.kt +++ b/src/test/kotlin/com/exactpro/th2/lwdataprovider/handlers/TestSearchMessagesHandler.kt @@ -556,7 +556,7 @@ internal class TestSearchMessagesHandler { decoder: Decoder, useTransportMode: Boolean ) = SearchMessagesHandler( - CradleMessageExtractor(manager, DummyDataMeasurement), + CradleMessageExtractor(manager, DummyDataMeasurement, false), decoder, executor, Configuration(