diff --git a/README.md b/README.md index c8e8c5c..f2a6488 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ dictionary: fix_dictionary.xml charset: US_ASCII dirtyMode: false decodeValuesToStrings: true +decodeComponentsToNestedMaps: true ``` #### beginString @@ -32,7 +33,10 @@ default value: `US_ASCII`. Charset for reading and writing FIX fields. default value: `false`. If `true`, processes all messages in dirty mode (generates warnings on invalid messages and continues processing). If `false`, only messages that contain the `encode-mode: dirty` property will be processed in dirty mode. #### decodeValuesToStrings -default value: `true`. Decode all values to strings instead of typed values. +default value: `true`. If `true`, decodes all values to strings instead of typed values. + +#### decodeComponentsToNestedMaps +default value: `true`. If `true`, decodes `components` to nested maps instead of unwrap component's map to message's main map. ## Release notes ### 0.1.0 diff --git a/src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodec.kt b/src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodec.kt index 581085a..91ab6b3 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodec.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodec.kt @@ -47,9 +47,10 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) private val charset = settings.charset private val isDirtyMode = settings.dirtyMode private val isDecodeToStrings = settings.decodeValuesToStrings + private val isDecodeComponentsToNestedMaps = settings.decodeComponentsToNestedMaps - private val fieldsEncode = convertToFieldsByName(dictionary.fields, true) - private val fieldsDecode = convertToFieldsByTag(dictionary.fields) + private val fieldsEncode = convertToFieldsByName(dictionary.fields, true, emptyList(), true) + private val fieldsDecode = convertToFieldsByTag(dictionary.fields, emptyList()) private val messagesByTypeForDecode: Map private val messagesByNameForEncode: Map @@ -169,6 +170,7 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) private fun Field.decode( source: ByteBuf, target: MutableMap, + tagsSet: MutableSet, value: String, tag: Int, isDirty: Boolean, @@ -190,23 +192,25 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) } } - // in dirty mode we'll use tag as field name if we have field duplication - val fieldName = if (isDirty && target.contains(name)) { - tag.toString() - } else { - name + if (!tagsSet.add(tag)) { + // we always handle tag duplication as error (isDirty = false) + handleError(false, context, "Duplicate $name field ($tag) with value: $value", value) } - val previous = target.put(fieldName, decodedValue) + val targetMap = if (isDecodeComponentsToNestedMaps) { + @Suppress("UNCHECKED_CAST") + path.fold(target) { map, key -> map.computeIfAbsent(key) { mutableMapOf() } as MutableMap } + } else { + target + } - // but even in dirty mode we can't write field if it's duplicated more than once - // because we use Map and it cant contain duplicates - check(previous == null) { "Duplicate $name field ($tag) with value: $value (previous: $previous)" } + targetMap[name] = decodedValue } private val prereadHeaderFields = arrayOf("BeginString", "BodyLength", "MsgType") private fun Message.decode(source: ByteBuf, bodyDef: Message, isDirty: Boolean, dictionaryFields: Map, context: IReportingContext): MutableMap = mutableMapOf().also { map -> + val tagsSet: MutableSet = hashSetOf() source.forEachField(charset, isDirty) { tag, value -> val field = get(tag) ?: if (isDirty) { when (this) { @@ -221,19 +225,27 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) dictField } else { context.warning(DIRTY_MODE_WARNING_PREFIX + "Field does not exist in dictionary. Field tag: $tag. Field value: $value.") - Primitive(false, tag.toString(), String::class.java, emptySet(), tag) + Primitive(false, tag.toString(), emptyList(), String::class.java, emptySet(), tag) } } else { // we reached next part of the message return@forEachField false } - field.decode(source, map, value, tag, isDirty, context) + field.decode(source, map, tagsSet, value, tag, isDirty, context) return@forEachField true } for (field in fields.values) { - if (field.isRequired && !map.contains(field.name) && field.name !in prereadHeaderFields) { + if (!field.isRequired) continue + + val tag = when (field) { + is Primitive -> field.tag + is Group -> field.counter + else -> error("Only `Primitive` and `Group` fields expected to be `required`") + } + + if (!tagsSet.contains(tag) && field.name !in prereadHeaderFields) { handleError(isDirty, context, "Required field missing. Field name: ${field.name}.") } } @@ -299,7 +311,7 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) source.forEachField(charset, isDirty) { tag, value -> val field = get(tag) ?: return@forEachField false - val group = if (tag == delimiter || !tags.add(tag) || map == null) { + val group = if (tag == delimiter || tags.contains(tag) || map == null) { if (tag != delimiter) { handleError(isDirty, context, "Field ${field.name} ($tag) appears before delimiter ($delimiter)") } @@ -313,7 +325,7 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) map ?: error("Group entry map can't be null.") } - field.decode(source, group, value, tag, isDirty, context) + field.decode(source, group, tags, value, tag, isDirty, context) return@forEachField true } @@ -432,11 +444,13 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) interface Field { val isRequired: Boolean val name: String + val path: List } data class Primitive( override val isRequired: Boolean, override val name: String, + override val path: List, val primitiveType: Class<*>, val values: Set, val tag: Int @@ -463,6 +477,7 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) data class Message( override val isRequired: Boolean, override val name: String, + override val path: List, val type: String, override val fields: Map, ) : Field, FieldMap() @@ -470,6 +485,7 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) data class Group( override val isRequired: Boolean, override val name: String, + override val path: List, val counter: Int, val delimiter: Int, override val fields: Map, @@ -536,42 +552,44 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) private val IFieldStructure.tag: Int get() = StructureUtils.getAttributeValue(this, "tag") - private fun IFieldStructure.toPrimitive(): Primitive = Primitive( - isRequired, + private fun IFieldStructure.toPrimitive(path: List, isRequiredParent: Boolean): Primitive = Primitive( + isRequiredParent && isRequired, name, + path, javaTypeToClass.getValue(javaType), values.values.map { it.getCastValue().toString() }.toSet(), tag ) - private fun convertToFieldsByName(fields: Map, isForEncode: Boolean): Map = linkedMapOf().apply { + private fun convertToFieldsByName(fields: Map, isForEncode: Boolean, path: List, isRequiredParent: Boolean): Map = linkedMapOf().apply { fields.forEach { (name, field) -> when { - field !is IMessageStructure -> this[name] = field.toPrimitive() - field.isGroup -> this[name] = field.toGroup(isForEncode) + field !is IMessageStructure -> this[name] = field.toPrimitive(path, isForEncode || isRequiredParent) + field.isGroup -> this[name] = field.toGroup(isForEncode, path, isForEncode || isRequiredParent) field.isComponent -> if (isForEncode) { - this[name] = field.toMessage(true) + this[name] = field.toMessage(true, path + name) } else { - this += convertToFieldsByName(field.fields, false) + this += convertToFieldsByName(field.fields, false, path + name, isRequiredParent && field.isRequired) } } } } - private fun convertToFieldsByTag(fields: Map): Map = linkedMapOf().apply { + private fun convertToFieldsByTag(fields: Map, path: List): Map = linkedMapOf().apply { fields.values.forEach { field -> when { - field !is IMessageStructure -> this[field.tag] = field.toPrimitive() - field.isGroup -> this[field.tag] = field.toGroup(false) - field.isComponent -> this += convertToFieldsByTag(field.fields) + field !is IMessageStructure -> this[field.tag] = field.toPrimitive(path, true) + field.isGroup -> this[field.tag] = field.toGroup(false, path, true) + field.isComponent -> this += convertToFieldsByTag(field.fields, path + field.name) } } } - private fun IMessageStructure.toMessage(isForEncode: Boolean): Message = Message( + private fun IMessageStructure.toMessage(isForEncode: Boolean, path: List): Message = Message( name = name, type = StructureUtils.getAttributeValue(this, FIELD_MESSAGE_TYPE) ?: name, - fields = convertToFieldsByName(this.fields, isForEncode), + fields = convertToFieldsByName(this.fields, isForEncode, path, !isComponent || isRequired), + path = path, isRequired = isRequired ) @@ -579,16 +597,17 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings) if (it is IMessageStructure && it.isComponent) getFirstTag(it) else it.tag } - private fun IMessageStructure.toGroup(isForEncode: Boolean): Group = Group( + private fun IMessageStructure.toGroup(isForEncode: Boolean, path: List, isRequiredParent: Boolean): Group = Group( name = name, counter = tag, delimiter = getFirstTag(this), - fields = convertToFieldsByName(this.fields, isForEncode), - isRequired = isRequired + fields = convertToFieldsByName(this.fields, isForEncode, emptyList(), true), + path = path, + isRequired = isRequiredParent && isRequired ) fun IDictionaryStructure.toMessages(isForEncode: Boolean): List = messages.values .filterNot { it.isGroup || it.isComponent } - .map { it.toMessage(isForEncode) } + .map { it.toMessage(isForEncode, emptyList()) } } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecSettings.kt b/src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecSettings.kt index 52f1825..3552807 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecSettings.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecSettings.kt @@ -25,4 +25,5 @@ data class FixNgCodecSettings( val charset: Charset = Charsets.US_ASCII, val dirtyMode: Boolean = false, val decodeValuesToStrings: Boolean = true, + val decodeComponentsToNestedMaps: Boolean = true ) : IPipelineCodecSettings \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecTest.kt b/src/test/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecTest.kt index 382613a..68240e2 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecTest.kt @@ -81,7 +81,7 @@ class FixNgCodecTest { @Test fun `decode with addition field that exists in dictionary`() { - expectedParsedBody["CFICode"] = "12345" + parsedBody["CFICode"] = "12345" decodeTest(MSG_ADDITIONAL_FIELD_DICT, "Unexpected field in message. Field name: CFICode. Field value: 12345.") } @@ -96,7 +96,7 @@ class FixNgCodecTest { @Test fun `decode with addition field that does not exists in dictionary`() { - expectedParsedBody["9999"] = "54321" + parsedBody["9999"] = "54321" decodeTest(MSG_ADDITIONAL_FIELD_NO_DICT, "Field does not exist in dictionary. Field tag: 9999. Field value: 54321.") } @@ -114,7 +114,7 @@ class FixNgCodecTest { @Test fun `decode with required field removed`() { - expectedParsedBody.remove("ExecID") + parsedBody.remove("ExecID") decodeTest(MSG_REQUIRED_FIELD_REMOVED, "Required field missing. Field name: ExecID.") } @@ -128,7 +128,7 @@ class FixNgCodecTest { @Test fun `decode with required delimiter field in group removed in first entry`() { @Suppress("UNCHECKED_CAST") - (expectedParsedBody["NoPartyIDs"] as MutableList>)[0].remove("PartyID") + ((parsedBody["TradingParty"] as Map)["NoPartyIDs"] as List>)[0].remove("PartyID") decodeTest(MSG_DELIMITER_FIELD_IN_GROUP_REMOVED_IN_FIRST_ENTRY, "Field PartyIDSource (447) appears before delimiter (448)") } @@ -142,7 +142,7 @@ class FixNgCodecTest { @Test fun `decode with required delimiter field in group removed in second entry`() { @Suppress("UNCHECKED_CAST") - (expectedParsedBody["NoPartyIDs"] as MutableList>)[1].remove("PartyID") + ((parsedBody["TradingParty"] as Map)["NoPartyIDs"] as List>)[1].remove("PartyID") decodeTest(MSG_DELIMITER_FIELD_IN_GROUP_REMOVED_IN_SECOND_ENTRY, "Field PartyIDSource (447) appears before delimiter (448)") } @@ -154,7 +154,7 @@ class FixNgCodecTest { @Test fun `decode with wrong enum value`() { - expectedParsedBody["ExecType"] = 'X' + parsedBody["ExecType"] = 'X' decodeTest(MSG_WRONG_ENUM, "Invalid value in enum field ExecType. Actual: X.") } @@ -178,7 +178,7 @@ class FixNgCodecTest { @Test fun `decode with wrong value type`() { - expectedParsedBody["LeavesQty"] = "Five" + parsedBody["LeavesQty"] = "Five" decodeTest(MSG_WRONG_TYPE, "Wrong number value in java.math.BigDecimal field 'LeavesQty'. Value: Five.") } @@ -190,7 +190,7 @@ class FixNgCodecTest { @Test fun `decode with empty value`() { - expectedParsedBody["Account"] = "" + parsedBody["Account"] = "" decodeTest(MSG_EMPTY_VAL, "Empty value in the field 'Account'.") } @@ -202,7 +202,7 @@ class FixNgCodecTest { @Test fun `decode with non printable characters`() { - expectedParsedBody["Account"] = "test\taccount" + parsedBody["Account"] = "test\taccount" decodeTest(MSG_NON_PRINTABLE, "Non printable characters in the field 'Account'.") } @@ -258,7 +258,7 @@ class FixNgCodecTest { private fun decodeTest( rawMessageString: String, expectedErrorText: String? = null, - expectedMessage: ParsedMessage = expectedParsedMessage, + expectedMessage: ParsedMessage = parsedMessage, dirtyMode: Boolean = true, decodeToStringValues: Boolean = false ) { @@ -328,7 +328,7 @@ class FixNgCodecTest { "OrderID" to "49415882", "ExecType" to '0', "OrdStatus" to '0', - "LeavesQty" to 500, + "LeavesQty" to BigDecimal(500), "CumQty" to BigDecimal(500), "SecurityID" to "NWDR", "SecurityIDSource" to "8", @@ -351,72 +351,17 @@ class FixNgCodecTest { "TimeInForce" to '0', "Side" to 'B', "Symbol" to "ABC", - "OrderQty" to 500, - "Price" to 1000, - "Unknown" to "500", - "TransactTime" to LocalDateTime.parse("2018-02-05T10:38:08.000008"), - "trailer" to mutableMapOf( - "CheckSum" to "191" - ) - ) - ) - - private val parsedBody: MutableMap = parsedMessage.body as MutableMap - - private val expectedParsedMessage = ParsedMessage( - MessageId("test_alias", Direction.OUTGOING, 0L, Instant.now(), emptyList()), - EventId("test_id", "test_book", "test_scope", Instant.now()), - "ExecutionReport", - mutableMapOf("encode-mode" to "dirty"), - PROTOCOL, - mutableMapOf( - "header" to mapOf( - "MsgSeqNum" to 10947, - "SenderCompID" to "SENDER", - "SendingTime" to LocalDateTime.parse("2023-04-19T10:36:07.415088"), - "TargetCompID" to "RECEIVER", - "BeginString" to "FIXT.1.1", - "BodyLength" to 295, - "MsgType" to "8" - ), - "ExecID" to "495504662", - "ClOrdID" to "zSuNbrBIZyVljs", - "OrigClOrdID" to "zSuNbrBIZyVljs", - "OrderID" to "49415882", - "ExecType" to '0', - "OrdStatus" to '0', - "LeavesQty" to BigDecimal(500), - "CumQty" to BigDecimal(500), - "SecurityID" to "NWDR", - "SecurityIDSource" to "8", - "NoPartyIDs" to listOf( - mapOf( - "PartyID" to "NGALL1FX01", - "PartyIDSource" to 'D', - "PartyRole" to 76 - ), - mapOf( - "PartyID" to "0", - "PartyIDSource" to 'P', - "PartyRole" to 3 - ) - ), - "Account" to "test", - "OrdType" to 'A', - "TimeInForce" to '0', - "Side" to 'B', - "Symbol" to "ABC", "OrderQty" to BigDecimal(500), "Price" to BigDecimal(1000), "Unknown" to "500", "TransactTime" to LocalDateTime.parse("2018-02-05T10:38:08.000008"), - "trailer" to mapOf( + "trailer" to mutableMapOf( "CheckSum" to "191" ) ) ) - private val expectedParsedBody: MutableMap = expectedParsedMessage.body as MutableMap + private val parsedBody: MutableMap = parsedMessage.body as MutableMap private val expectedMessageWithoutBody = ParsedMessage( MessageId("test_alias", Direction.OUTGOING, 0L, Instant.now(), emptyList()),