Skip to content

Commit

Permalink
[TH2-5121] Fix JSON escaping for custom serialization (#75)
Browse files Browse the repository at this point in the history
* Add proper JSON escaping for custom serialization

* Update version and readme
  • Loading branch information
OptimumCode authored Nov 8, 2023
1 parent 6e54dbc commit 08d597d
Show file tree
Hide file tree
Showing 6 changed files with 117 additions and 17 deletions.
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,12 @@ spec:
# Release notes:
## 2.4.1
### Fixed:
+ custom JSON serialization did not do proper escaping for strings
## 2.4.0
+ Added `batchSizeBytes` parameter to limit batch size by size in bytes rather than count of messages.
Expand Down
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

kotlin.code.style=official

release_version=2.4.0
release_version=2.4.1
description='th2 Lightweight data provider component'
vcs_url=https://github.com/th2-net/th2-lw-data-provider
docker_image_name=
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ abstract class AbstractJsonFormatter : JsonFormatter {
private fun isNeedToEscape(s: String): Boolean {
// ascii 32 is space, all chars below should be escaped
return s.chars()
.anyMatch { it < 32 || it == CustomProtoJsonFormatter.QUOTE_CHAR || it == CustomProtoJsonFormatter.BACK_SLASH }
.anyMatch { it < 32 || it == QUOTE_CHAR || it == BACK_SLASH }
}

protected fun convertStringToJson(s: String, builder: StringBuilder) {
Expand Down Expand Up @@ -233,4 +233,9 @@ abstract class AbstractJsonFormatter : JsonFormatter {
}
sb.append("}")
}

companion object {
internal const val QUOTE_CHAR = '"'.code
internal const val BACK_SLASH = '\\'.code
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,6 @@ import com.exactpro.th2.common.grpc.Value

class CustomProtoJsonFormatter : AbstractJsonFormatter() {

companion object {
internal const val QUOTE_CHAR = '"'.code
internal const val BACK_SLASH = '\\'.code
}

override fun printV (value: Value, sb: StringBuilder) {
when (value.kindCase) {
Value.KindCase.SIMPLE_VALUE -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,16 @@
package com.exactpro.th2.lwdataprovider.entities.responses

import com.exactpro.cradle.messages.StoredMessageId
import com.exactpro.cradle.utils.EscapeUtils.escape
import com.exactpro.cradle.utils.TimeUtils
import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage
import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.toByteArray
import com.exactpro.th2.lwdataprovider.entities.responses.ser.numberOfDigits
import io.javalin.http.util.JsonEscapeUtil
import java.io.ByteArrayOutputStream
import java.io.OutputStream
import java.time.Instant
import kotlin.math.ceil
import kotlin.math.log10
import kotlin.math.max
import kotlin.text.Charsets.UTF_8
import com.exactpro.cradle.utils.EscapeUtils.escape as cradleEscape

private val COMMA = ",".toByteArray(UTF_8).first().toInt()
private val COLON = ":".toByteArray(UTF_8).first().toInt()
Expand All @@ -54,6 +52,7 @@ private val MESSAGE_TYPE_FILED = """"messageType"""".toByteArray(UTF_8)
private val PROPERTIES_FILED = """"properties"""".toByteArray(UTF_8)
private val PROTOCOL_FILED = """"protocol"""".toByteArray(UTF_8)
private val FIELDS_FILED = """"fields"""".toByteArray(UTF_8)
private val ESCAPE_CHARACTERS = charArrayOf('\"', '\n', '\r', '\\', '\t', '\b')

fun ProviderMessage53Transport.toJSONByteArray(): ByteArray =
ByteArrayOutputStream(1_024 * 2).apply { // TODO: init size
Expand All @@ -80,14 +79,22 @@ fun ProviderMessage53Transport.toJSONByteArray(): ByteArray =
write(CLOSING_CURLY_BRACE)
}.toByteArray()

private fun jsonEscape(value: String): String {
return if (ESCAPE_CHARACTERS.any(value::contains)) {
JsonEscapeUtil.escape(value)
} else {
value
}
}

private fun OutputStream.writeMessageId(messageId: StoredMessageId) {
write(MESSAGE_ID_FILED)
write(COLON)
write(DOUBLE_QUOTE)
with(messageId) {
write(escape(bookId.toString()).toByteArray(UTF_8))
write(jsonEscape(cradleEscape(bookId.toString())).toByteArray(UTF_8))
write(COLON)
write(escape(sessionAlias).toByteArray(UTF_8))
write(jsonEscape(cradleEscape(sessionAlias)).toByteArray(UTF_8))
write(COLON)
write(direction.label.toByteArray(UTF_8))
write(COLON)
Expand Down Expand Up @@ -166,7 +173,7 @@ private fun OutputStream.writeAttachedEventIds(attachedEventIds: Set<String>) {
write(COMMA)
}
write(DOUBLE_QUOTE)
write(escape(eventId).toByteArray(UTF_8))
write(jsonEscape(eventId).toByteArray(UTF_8))
write(DOUBLE_QUOTE)
}
write(CLOSING_SQUARE_BRACE)
Expand Down Expand Up @@ -208,7 +215,7 @@ private fun OutputStream.writeFieldWithoutEscaping(name: ByteArray, value: Strin
write(DOUBLE_QUOTE)
}

private fun OutputStream.writeField(name: ByteArray, value: String) = writeFieldWithoutEscaping(name, escape(value))
private fun OutputStream.writeField(name: ByteArray, value: String) = writeFieldWithoutEscaping(name, jsonEscape(value))
private fun OutputStream.writeField(name: ByteArray, value: Number) {
write(name)
write(COLON)
Expand All @@ -217,11 +224,11 @@ private fun OutputStream.writeField(name: ByteArray, value: Number) {

private fun OutputStream.writeField(name: String, value: String) {
write(DOUBLE_QUOTE)
write(escape(name).toByteArray(UTF_8))
write(jsonEscape(name).toByteArray(UTF_8))
write(DOUBLE_QUOTE)
write(COLON)
write(DOUBLE_QUOTE)
write(escape(value).toByteArray(UTF_8))
write(jsonEscape(value).toByteArray(UTF_8))
write(DOUBLE_QUOTE)
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Copyright 2023 Exactpro (Exactpro Systems Limited)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.exactpro.th2.lwdataprovider.entities.responses

import com.exactpro.cradle.BookId
import com.exactpro.cradle.messages.StoredMessageId
import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.EventId
import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageId
import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage
import com.exactpro.th2.lwdataprovider.entities.internal.Direction
import com.fasterxml.jackson.databind.json.JsonMapper
import io.netty.buffer.Unpooled
import org.junit.jupiter.api.assertDoesNotThrow
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.ValueSource
import java.time.Instant
import java.util.Base64

internal class TestCustomSerializerKt {
private val mapper = JsonMapper()
@ParameterizedTest(name = "char `{0}` does not cause problems")
@ValueSource(chars = ['\"', '\n', '\r', '\\', '\t', '\b', ':'])
fun `writes valid json`(escapeCharacter: Char) {
val timestamp = Instant.now()
val message = ProviderMessage53Transport(
timestamp = timestamp,
direction = Direction.OUT,
sessionId = "ses${escapeCharacter}sion",
attachedEventIds = setOf(
"eve${escapeCharacter}nt",
),
bodyBase64 = Base64.getEncoder().encodeToString(byteArrayOf(42, 43)),
messageId = StoredMessageId(
BookId("bo${escapeCharacter}ok"),
"session${escapeCharacter}Alias",
com.exactpro.cradle.Direction.SECOND,
timestamp,
42L,
),
body = listOf(
TransportMessageContainer(
sessionGroup = "session${escapeCharacter}Group",
parsedMessage = ParsedMessage(
id = MessageId(
sessionAlias = "session${escapeCharacter}Alias",
direction = com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.Direction.OUTGOING,
sequence = 42L,
timestamp = timestamp,
),
eventId = EventId(
id = "eve${escapeCharacter}nt",
scope = "scop${escapeCharacter}e",
timestamp = timestamp,
book = "bo${escapeCharacter}ok",
),
type = "Message${escapeCharacter}Type",
metadata = mapOf(
"ke${escapeCharacter}y" to "val${escapeCharacter}ue",
),
protocol = "proto${escapeCharacter}col",
rawBody = Unpooled.wrappedBuffer(
"""{"test":42}""".toByteArray(Charsets.UTF_8)
),
)
),
),
)

val jsonBytes = message.toJSONByteArray()

assertDoesNotThrow { mapper.readTree(jsonBytes) }
}
}

0 comments on commit 08d597d

Please sign in to comment.