diff --git a/README.md b/README.md index 239d4280..490cc230 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,12 @@ spec: # Release notes: +## 2.4.1 + +### Fixed: + ++ custom JSON serialization did not do proper escaping for strings + ## 2.4.0 + Added `batchSizeBytes` parameter to limit batch size by size in bytes rather than count of messages. diff --git a/gradle.properties b/gradle.properties index 92114848..6863c75e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,7 +16,7 @@ kotlin.code.style=official -release_version=2.4.0 +release_version=2.4.1 description='th2 Lightweight data provider component' vcs_url=https://github.com/th2-net/th2-lw-data-provider docker_image_name= diff --git a/src/main/kotlin/com/exactpro/th2/lwdataprovider/AbstractJsonFormatter.kt b/src/main/kotlin/com/exactpro/th2/lwdataprovider/AbstractJsonFormatter.kt index 2de39197..98510453 100644 --- a/src/main/kotlin/com/exactpro/th2/lwdataprovider/AbstractJsonFormatter.kt +++ b/src/main/kotlin/com/exactpro/th2/lwdataprovider/AbstractJsonFormatter.kt @@ -119,7 +119,7 @@ abstract class AbstractJsonFormatter : JsonFormatter { private fun isNeedToEscape(s: String): Boolean { // ascii 32 is space, all chars below should be escaped return s.chars() - .anyMatch { it < 32 || it == CustomProtoJsonFormatter.QUOTE_CHAR || it == CustomProtoJsonFormatter.BACK_SLASH } + .anyMatch { it < 32 || it == QUOTE_CHAR || it == BACK_SLASH } } protected fun convertStringToJson(s: String, builder: StringBuilder) { @@ -233,4 +233,9 @@ abstract class AbstractJsonFormatter : JsonFormatter { } sb.append("}") } + + companion object { + internal const val QUOTE_CHAR = '"'.code + internal const val BACK_SLASH = '\\'.code + } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/lwdataprovider/CustomProtoJsonFormatter.kt b/src/main/kotlin/com/exactpro/th2/lwdataprovider/CustomProtoJsonFormatter.kt index c5c93760..59b7d1e5 100644 --- a/src/main/kotlin/com/exactpro/th2/lwdataprovider/CustomProtoJsonFormatter.kt +++ b/src/main/kotlin/com/exactpro/th2/lwdataprovider/CustomProtoJsonFormatter.kt @@ -20,11 +20,6 @@ import com.exactpro.th2.common.grpc.Value class CustomProtoJsonFormatter : AbstractJsonFormatter() { - companion object { - internal const val QUOTE_CHAR = '"'.code - internal const val BACK_SLASH = '\\'.code - } - override fun printV (value: Value, sb: StringBuilder) { when (value.kindCase) { Value.KindCase.SIMPLE_VALUE -> { diff --git a/src/main/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/CustomSerilizer.kt b/src/main/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/CustomSerializer.kt similarity index 91% rename from src/main/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/CustomSerilizer.kt rename to src/main/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/CustomSerializer.kt index 770614fe..7cffa863 100644 --- a/src/main/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/CustomSerilizer.kt +++ b/src/main/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/CustomSerializer.kt @@ -17,18 +17,16 @@ package com.exactpro.th2.lwdataprovider.entities.responses import com.exactpro.cradle.messages.StoredMessageId -import com.exactpro.cradle.utils.EscapeUtils.escape import com.exactpro.cradle.utils.TimeUtils import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.toByteArray import com.exactpro.th2.lwdataprovider.entities.responses.ser.numberOfDigits +import io.javalin.http.util.JsonEscapeUtil import java.io.ByteArrayOutputStream import java.io.OutputStream import java.time.Instant -import kotlin.math.ceil -import kotlin.math.log10 -import kotlin.math.max import kotlin.text.Charsets.UTF_8 +import com.exactpro.cradle.utils.EscapeUtils.escape as cradleEscape private val COMMA = ",".toByteArray(UTF_8).first().toInt() private val COLON = ":".toByteArray(UTF_8).first().toInt() @@ -54,6 +52,7 @@ private val MESSAGE_TYPE_FILED = """"messageType"""".toByteArray(UTF_8) private val PROPERTIES_FILED = """"properties"""".toByteArray(UTF_8) private val PROTOCOL_FILED = """"protocol"""".toByteArray(UTF_8) private val FIELDS_FILED = """"fields"""".toByteArray(UTF_8) +private val ESCAPE_CHARACTERS = charArrayOf('\"', '\n', '\r', '\\', '\t', '\b') fun ProviderMessage53Transport.toJSONByteArray(): ByteArray = ByteArrayOutputStream(1_024 * 2).apply { // TODO: init size @@ -80,14 +79,22 @@ fun ProviderMessage53Transport.toJSONByteArray(): ByteArray = write(CLOSING_CURLY_BRACE) }.toByteArray() +private fun jsonEscape(value: String): String { + return if (ESCAPE_CHARACTERS.any(value::contains)) { + JsonEscapeUtil.escape(value) + } else { + value + } +} + private fun OutputStream.writeMessageId(messageId: StoredMessageId) { write(MESSAGE_ID_FILED) write(COLON) write(DOUBLE_QUOTE) with(messageId) { - write(escape(bookId.toString()).toByteArray(UTF_8)) + write(jsonEscape(cradleEscape(bookId.toString())).toByteArray(UTF_8)) write(COLON) - write(escape(sessionAlias).toByteArray(UTF_8)) + write(jsonEscape(cradleEscape(sessionAlias)).toByteArray(UTF_8)) write(COLON) write(direction.label.toByteArray(UTF_8)) write(COLON) @@ -166,7 +173,7 @@ private fun OutputStream.writeAttachedEventIds(attachedEventIds: Set) { write(COMMA) } write(DOUBLE_QUOTE) - write(escape(eventId).toByteArray(UTF_8)) + write(jsonEscape(eventId).toByteArray(UTF_8)) write(DOUBLE_QUOTE) } write(CLOSING_SQUARE_BRACE) @@ -208,7 +215,7 @@ private fun OutputStream.writeFieldWithoutEscaping(name: ByteArray, value: Strin write(DOUBLE_QUOTE) } -private fun OutputStream.writeField(name: ByteArray, value: String) = writeFieldWithoutEscaping(name, escape(value)) +private fun OutputStream.writeField(name: ByteArray, value: String) = writeFieldWithoutEscaping(name, jsonEscape(value)) private fun OutputStream.writeField(name: ByteArray, value: Number) { write(name) write(COLON) @@ -217,11 +224,11 @@ private fun OutputStream.writeField(name: ByteArray, value: Number) { private fun OutputStream.writeField(name: String, value: String) { write(DOUBLE_QUOTE) - write(escape(name).toByteArray(UTF_8)) + write(jsonEscape(name).toByteArray(UTF_8)) write(DOUBLE_QUOTE) write(COLON) write(DOUBLE_QUOTE) - write(escape(value).toByteArray(UTF_8)) + write(jsonEscape(value).toByteArray(UTF_8)) write(DOUBLE_QUOTE) } diff --git a/src/test/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/TestCustomSerializerKt.kt b/src/test/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/TestCustomSerializerKt.kt new file mode 100644 index 00000000..1332e026 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/lwdataprovider/entities/responses/TestCustomSerializerKt.kt @@ -0,0 +1,87 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.lwdataprovider.entities.responses + +import com.exactpro.cradle.BookId +import com.exactpro.cradle.messages.StoredMessageId +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.EventId +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageId +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage +import com.exactpro.th2.lwdataprovider.entities.internal.Direction +import com.fasterxml.jackson.databind.json.JsonMapper +import io.netty.buffer.Unpooled +import org.junit.jupiter.api.assertDoesNotThrow +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import java.time.Instant +import java.util.Base64 + +internal class TestCustomSerializerKt { + private val mapper = JsonMapper() + @ParameterizedTest(name = "char `{0}` does not cause problems") + @ValueSource(chars = ['\"', '\n', '\r', '\\', '\t', '\b', ':']) + fun `writes valid json`(escapeCharacter: Char) { + val timestamp = Instant.now() + val message = ProviderMessage53Transport( + timestamp = timestamp, + direction = Direction.OUT, + sessionId = "ses${escapeCharacter}sion", + attachedEventIds = setOf( + "eve${escapeCharacter}nt", + ), + bodyBase64 = Base64.getEncoder().encodeToString(byteArrayOf(42, 43)), + messageId = StoredMessageId( + BookId("bo${escapeCharacter}ok"), + "session${escapeCharacter}Alias", + com.exactpro.cradle.Direction.SECOND, + timestamp, + 42L, + ), + body = listOf( + TransportMessageContainer( + sessionGroup = "session${escapeCharacter}Group", + parsedMessage = ParsedMessage( + id = MessageId( + sessionAlias = "session${escapeCharacter}Alias", + direction = com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.Direction.OUTGOING, + sequence = 42L, + timestamp = timestamp, + ), + eventId = EventId( + id = "eve${escapeCharacter}nt", + scope = "scop${escapeCharacter}e", + timestamp = timestamp, + book = "bo${escapeCharacter}ok", + ), + type = "Message${escapeCharacter}Type", + metadata = mapOf( + "ke${escapeCharacter}y" to "val${escapeCharacter}ue", + ), + protocol = "proto${escapeCharacter}col", + rawBody = Unpooled.wrappedBuffer( + """{"test":42}""".toByteArray(Charsets.UTF_8) + ), + ) + ), + ), + ) + + val jsonBytes = message.toJSONByteArray() + + assertDoesNotThrow { mapper.readTree(jsonBytes) } + } +} \ No newline at end of file