diff --git a/.github/workflows/dev-docker-publish.yml b/.github/workflows/dev-docker-publish.yml index dbf999d..32a4535 100644 --- a/.github/workflows/dev-docker-publish.yml +++ b/.github/workflows/dev-docker-publish.yml @@ -5,42 +5,15 @@ on: branches-ignore: - master - version-* - + - dependabot** + paths-ignore: + - README.md jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - # Prepare custom build version - - name: Get branch name - id: branch - run: echo ::set-output name=branch_name::${GITHUB_REF#refs/*/} - - name: Get release_version - id: ver - uses: christian-draeger/read-properties@1.0.1 - with: - path: gradle.properties - property: release_version - - name: Build custom release version - id: release_ver - run: echo ::set-output name=value::"${{ steps.ver.outputs.value }}-${{ steps.branch.outputs.branch_name }}-${{ github.run_id }}" - - name: Show custom release version - run: echo ${{ steps.release_ver.outputs.value }} - # Build and publish image - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} - - run: echo "::set-output name=REPOSITORY_NAME::$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" - id: meta - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${{ steps.release_ver.outputs.value }} - labels: com.exactpro.th2.${{ steps.meta.outputs.REPOSITORY_NAME }}=${{ steps.ver.outputs.value }} \ No newline at end of file + build-job: + uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main + with: + build-target: 'Docker' + docker-username: ${{ github.actor }} + secrets: + docker-password: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9f51020..291d3ff 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,31 +7,13 @@ on: - version-* paths: - gradle.properties +# - package_info.json jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} - - run: echo "::set-output name=REPOSITORY_NAME::$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" - id: meta - - name: Read version from gradle.properties - id: read_property - uses: christian-draeger/read-properties@1.0.1 - with: - path: ./gradle.properties - property: release_version - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${{ steps.read_property.outputs.value }} - labels: com.exactpro.th2.${{ steps.meta.outputs.REPOSITORY_NAME }}=${{ steps.read_property.outputs.value }} \ No newline at end of file + build-job: + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Docker' + docker-username: ${{ github.actor }} + secrets: + docker-password: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 932d612..4fb0ed7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# Codec Xml via Xsd -![version](https://img.shields.io/badge/version-0.0.3-blue.svg) +# Codec Xml via Xsd (0.0.5) +![version](https://img.shields.io/badge/version-0.0.4-blue.svg) # How it works: @@ -57,13 +57,18 @@ Error from validation process can be disabled for test purposes by `dirtyValidat ### Configuration example * typePointer - Path to message type value for decode (null by default) +* encodeValidation - This flag determines if messages are validated +during encoding (false by default). Note: this validation will significantly +slow down the performance and requires an archive of XSDs + +## Temporary disabled * dirtyValidation - Disable/enable error during validation phase. If disabled all errors will be only visible in log (false by default) * expectsDeclaration - Disable/enable validation of declaration - is it exist or not (true by default) ```yaml typePointer: /root/node/node2/type -dirtyValidation: false -expectsDeclaration: false +#dirtyValidation: false +#expectsDeclaration: false ``` For example: @@ -77,7 +82,7 @@ spec: custom-config: codecSettings: typePointer: /root/node/node2/type - dirtyValidation: false +# dirtyValidation: false ``` ## Required pins @@ -142,6 +147,18 @@ spec: ## Changelog +### v0.0.5 + +* Migrate to StAX parser +* Migrated from common:3.31.5 to common:3.40.0 +* Migrated from bom:3.1.0 to common:3.2.0 + +### v0.0.4 + +#### Feature: + +* Self-closing tags are ignored by the codec + ### v0.0.3 #### Feature: @@ -159,4 +176,4 @@ spec: #### Feature: -* First realization using underscore library as parser for json \ No newline at end of file +* First realization using underscore library as parser for json diff --git a/build.gradle b/build.gradle index ce1ccfa..1a1d6f5 100755 --- a/build.gradle +++ b/build.gradle @@ -22,7 +22,6 @@ plugins { ext { sharedDir = file("${project.rootDir}/shared") - sailfishVersion = '3.2.1752' } group = 'com.exactpro.th2' @@ -67,22 +66,22 @@ jar { } dependencies { - api platform('com.exactpro.th2:bom:3.1.0') - implementation 'com.exactpro.th2:common:3.31.5' - implementation 'com.exactpro.th2:codec:4.2.0' - implementation 'com.exactpro.th2:sailfish-utils:3.12.3' - - implementation "com.exactpro.sf:sailfish-core:${sailfishVersion}" - - implementation 'org.slf4j:slf4j-log4j12' - implementation 'org.slf4j:slf4j-api' + api platform('com.exactpro.th2:bom:3.2.0') + implementation 'com.exactpro.th2:common:3.44.0' + implementation 'com.exactpro.th2:codec:4.8.0' implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}" implementation "org.jetbrains.kotlin:kotlin-reflect:${kotlin_version}" + implementation 'io.github.microutils:kotlin-logging:2.1.21' implementation 'com.fasterxml.jackson.core:jackson-core:2.13.1' implementation 'com.github.javadev:underscore:1.69' - + + implementation group: 'org.apache.ws.xmlschema', name: 'xmlschema-core', version: '2.3.0' + + implementation 'org.slf4j:slf4j-api:2.0.6' + + testImplementation 'org.slf4j:slf4j-log4j12:2.0.6' testImplementation "org.jetbrains.kotlin:kotlin-test-junit5:${kotlin_version}" testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}" testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" diff --git a/gradle.properties b/gradle.properties index 6326cda..0a1ed78 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,8 +13,8 @@ # limitations under the License. # -release_version=0.0.3 -kotlin_version=1.5.31 +release_version=0.0.5 +kotlin_version=1.5.32 description = 'th2 codec xml via xsd' vcs_url=https://github.com/th2-net/th2-codec-xml-via-xsd \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/NodeContent.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/NodeContent.kt new file mode 100644 index 0000000..dcdc69f --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/NodeContent.kt @@ -0,0 +1,145 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.codec.xml + +import com.exactpro.th2.common.grpc.Message +import com.exactpro.th2.common.grpc.MessageMetadata +import com.exactpro.th2.common.grpc.Value +import com.exactpro.th2.common.message.addField +import com.exactpro.th2.common.value.toValue +import javax.xml.namespace.QName + +class NodeContent( + private val nodeName: QName, + decorator: XmlCodecStreamReader +) { + private val messageBuilder = Message.newBuilder().apply { + metadata = MessageMetadata.getDefaultInstance() //FIXME: remove + } + private val textSB = StringBuilder() + + private val childNodes: MutableMap> = mutableMapOf() + private var isMessage: Boolean = false + private val isEmpty: Boolean + get() = !isMessage && textSB.isEmpty() + + val name: String = nodeName.toNodeName() + + init { + decorator.namespaceCount.let { size -> + if (size > 0) { + for (i in 0 until size) { + decorator.getNamespaceURI(i).also { value -> + val uri = value ?: "" + val prefix = decorator.namespaceContext.getPrefix(uri) ?: "" + + messageBuilder.addField(makeFieldName(NAMESPACE, prefix, true), uri) + } + } + isMessage = true + } + } + decorator.attributeCount.let { size -> + if (size > 0) { + for (i in 0 until size) { + val localPart = decorator.getAttributeLocalName(i) + val prefix = decorator.getAttributePrefix(i) + + messageBuilder.addField(makeFieldName(prefix, localPart, true), decorator.getAttributeValue(i)) + } + isMessage = true + } + } + } + + fun putChild(name: QName, node: NodeContent) { + childNodes.compute(name) { _, value -> + value + ?.apply { add(node) } + ?: mutableListOf(node) + } + isMessage = true + } + + fun appendText(text: String) { + if (text.isNotBlank()) { + textSB.append(text) + } + } + + fun release() { + if (isMessage) { + childNodes.forEach { (name, values) -> + try { + val notEmptyValues = values.filterNot(NodeContent::isEmpty) + + if (notEmptyValues.isNotEmpty()) { + val first = notEmptyValues.first() + when (values.size) { // Clarefy type of element: list or single + 0 -> error("Sub element $name hasn't got values") + 1 -> messageBuilder.addField(first.name, first.toValue()) + else -> messageBuilder.addField( + first.name, + notEmptyValues.asSequence() + .map(NodeContent::toValue) + .toListValue() + ) + } + } + } catch (e: RuntimeException) { + throw IllegalStateException("The `$name` field can't be released in the `$nodeName` node", e) + } + } + if(textSB.isNotBlank()) { + messageBuilder.addField(TEXT_FIELD_NAME, textSB.toValue()) + } + } + } + + fun toMessage(): Message { + check(isMessage) { + "The $nodeName node isn't message" + } + return messageBuilder.build() + } + + override fun toString(): String { + return "NodeContent(nodeName=$nodeName, childNodes=$childNodes, text=$textSB)" + } + + private fun Sequence.toListValue(): Value = Value.newBuilder().apply { + listValueBuilder.apply { + forEach(::addValues) + } + }.build() + + private fun toValue(): Value = if (isMessage) { + messageBuilder.toValue() + } else { + textSB.toValue() + } + + companion object { + private const val NAMESPACE = "xmlns" + private const val TEXT_FIELD_NAME = "#text" + + private fun QName.toNodeName(): String = makeFieldName(prefix, localPart) + + private fun makeFieldName(first: String, second: String, isAttribute: Boolean = false): String { + return "${if (isAttribute) "-" else ""}$first${if (first.isNotBlank() && second.isNotBlank()) ":" else ""}$second" + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlCodecStreamReader.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlCodecStreamReader.kt new file mode 100644 index 0000000..54dbff7 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlCodecStreamReader.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.codec.xml + +import com.exactpro.th2.common.grpc.Message +import com.exactpro.th2.common.grpc.RawMessage +import com.exactpro.th2.common.grpc.Value +import com.exactpro.th2.common.message.addField +import com.exactpro.th2.common.message.direction +import com.exactpro.th2.common.message.getField +import com.exactpro.th2.common.message.message +import com.exactpro.th2.common.message.sequence +import com.exactpro.th2.common.message.sessionAlias +import com.exactpro.th2.common.value.toValue +import com.google.protobuf.TextFormat +import java.io.ByteArrayInputStream +import java.util.* +import javax.xml.stream.XMLInputFactory +import javax.xml.stream.XMLStreamException +import javax.xml.stream.util.StreamReaderDelegate + +class XmlCodecStreamReader( + private val rawMessage: RawMessage, + private val pointer: List +) + : StreamReaderDelegate( + XML_INPUT_FACTORY.createXMLStreamReader( + ByteArrayInputStream(rawMessage.body.toByteArray()) + ) +) { + private lateinit var rootNode: NodeContent + private lateinit var messageType: String + + private val elements = Stack() + + @Throws(XMLStreamException::class) + override fun next(): Int = super.next().also { eventCode -> + when (eventCode) { + START_ELEMENT -> { + val qName = name + NodeContent(qName, this).also { nodeContent -> + if (elements.isNotEmpty()) { + elements.peek() + .putChild(qName, nodeContent) + } + + elements.push(nodeContent) + + if (!this::messageType.isInitialized) { + messageType = localName + } + if (!this::rootNode.isInitialized) { + rootNode = nodeContent + } + } + } + CHARACTERS -> elements.peek().appendText(text) + END_ELEMENT -> elements.pop().also(NodeContent::release) + } + } + + fun getMessage(): Message { + check(elements.isEmpty()) { + "Some of XML nodes of ${TextFormat.shortDebugString(rawMessage.metadata.id)} message aren't closed ${elements.joinToString { it.name }}" + } + + return message().apply { + val rawMetadata = rawMessage.metadata + + if (rawMessage.hasParentEventId()) { + parentEventId = rawMessage.parentEventId + } + + addField(rootNode.name, rootNode.toMessage().toValue()) + + metadataBuilder.apply { + putAllProperties(rawMetadata.propertiesMap) + id = rawMetadata.id + timestamp = rawMetadata.timestamp + protocol = rawMetadata.protocol + messageType = extractMessageType(this@XmlCodecStreamReader.messageType) + } + }.build() + } + + private fun Message.Builder.extractMessageType(defaultMessageType: String): String { + if (pointer.isEmpty()) return defaultMessageType + + var currentNode = this.toValue() + pointer.forEachIndexed { index, element -> + check(currentNode.hasMessageValue()) { + "The `${pointer.take(index)}` node (${currentNode.kindCase}) isn't message in the th2 message $sessionAlias:$direction$sequence" + } + currentNode = requireNotNull(currentNode.messageValue.getField(element)) { + "The `${pointer.take(index + 1)}` element isn't found in message $sessionAlias:$direction$sequence" + } + } + check(currentNode.kindCase == Value.KindCase.SIMPLE_VALUE) { + "The `$pointer` node (${currentNode.kindCase}) isn't simple value in the th2 message $sessionAlias:$direction$sequence" + } + + return currentNode.simpleValue + } + + companion object { + private val XML_INPUT_FACTORY = XMLInputFactory.newInstance() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodec.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodec.kt index baf975a..89ee8f7 100755 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodec.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodec.kt @@ -18,7 +18,6 @@ package com.exactpro.th2.codec.xml import com.exactpro.th2.codec.DecodeException import com.exactpro.th2.codec.api.IPipelineCodec import com.exactpro.th2.codec.xml.utils.toMap -import com.exactpro.th2.codec.xml.utils.toProto import com.exactpro.th2.codec.xml.xsd.XsdValidator import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.Message @@ -34,11 +33,15 @@ import org.slf4j.LoggerFactory import java.nio.charset.Charset import java.nio.file.Path -open class XmlPipelineCodec(private val settings: XmlPipelineCodecSettings, xsdMap: Map) : IPipelineCodec { +open class XmlPipelineCodec(settings: XmlPipelineCodecSettings, xsdMap: Map = mapOf()) : IPipelineCodec { - private val pointer = settings.typePointer?.split("/")?.filterNot { it.isBlank() } + private val pointer = settings.typePointer + ?.split("/")?.filter(String::isNotBlank) + ?: listOf() private var xmlCharset: Charset = Charsets.UTF_8 - private val validator = XsdValidator(xsdMap, settings.dirtyValidation) +// private val oldValidator = XsdValidator(xsdMap, settings.dirtyValidation) + private val encodeValidation = settings.encodeValidation + private val oldValidator = XsdValidator(xsdMap, false) override fun encode(messageGroup: MessageGroup): MessageGroup { val messages = messageGroup.messagesList @@ -65,8 +68,10 @@ open class XmlPipelineCodec(private val settings: XmlPipelineCodecSettings, xsdM val map = message.toMap() val xmlString = Xml.toXml(map) - validator.validate(xmlString.toByteArray()) - LOGGER.debug("Validation of incoming parsed message complete: ${message.messageType}") + if (encodeValidation) { + oldValidator.validate(xmlString.toByteArray()) + LOGGER.debug("Validation of incoming parsed message complete: ${message.messageType}") + } return RawMessage.newBuilder().apply { if (message.hasParentEventId()) parentEventId = message.parentEventId @@ -102,60 +107,25 @@ open class XmlPipelineCodec(private val settings: XmlPipelineCodecSettings, xsdM private fun decodeOne(rawMessage: RawMessage): Message { try { - validator.validate(rawMessage.body.toByteArray()) - LOGGER.debug("Validation of incoming raw message complete: ${rawMessage.logId}") - val xmlString = rawMessage.body.toStringUtf8() - @Suppress("UNCHECKED_CAST") - val map = Xml.fromXml(xmlString) as MutableMap - - LOGGER.trace("Result of the 'Xml.fromXml' method is ${map.keys} for $xmlString") - map -= STANDALONE - map -= ENCODING - - if (OMIT_XML_DECLARATION in map) { - // U library will tell by this option is there no declaration - check(!settings.expectsDeclaration || map[OMIT_XML_DECLARATION] == NO) { "Expecting declaration inside xml data" } - map -= OMIT_XML_DECLARATION - } + val reader = XmlCodecStreamReader(rawMessage, pointer) - if (map.size > 1) { - error("There was more than one root node in processed xml, result json has [${map.size}]: ${map.keys.joinToString(", ")}") + try { + while (reader.hasNext()) { reader.next() } + return reader.getMessage() + } finally { + reader.close() } - - val msgType: String = pointer?.let { map.getNode(it) } ?: map.keys.first() - - return map.toProto(msgType, rawMessage) } catch (e: Exception) { - throw DecodeException("Can not decode message. Can not parse XML. ${rawMessage.body.toStringUtf8()}", e) - } - } - - private inline fun Map<*,*>.getNode(pointer: List): T { - var current: Any = this - - for (name in pointer) { - current = (current as? Map<*, *>)?.get(name) ?: error("Can not find element by name '$name' in path: $pointer") + throw DecodeException("Can not decode message ${rawMessage.logId}. Can not parse XML. ${rawMessage.body.toStringUtf8()}", e) } - return current as T } - companion object { private val LOGGER: Logger = LoggerFactory.getLogger(XmlPipelineCodec::class.java) - private const val NO = "no" - - /** - * The constant from [Xml.OMITXMLDECLARATION] - */ - private const val OMIT_XML_DECLARATION = "#omit-xml-declaration" - /** - * The constant from [Xml.ENCODING] - */ - private const val ENCODING = "#encoding" - /** - * The constant from [Xml.STANDALONE] - */ - private const val STANDALONE = "#standalone" +// private val SCHEMA_FACTORY = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI).apply { +// errorHandler = XsdErrorHandler() +// } + } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecFactory.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecFactory.kt index f6f000f..466b20e 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecFactory.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecFactory.kt @@ -31,10 +31,11 @@ class XmlPipelineCodecFactory : IPipelineCodecFactory { lateinit var xsdMap: Map override fun init(dictionary: InputStream) { - xsdMap = decodeInputToDictionary(dictionary, XSD_FOLDER) - if (xsdMap.isEmpty()) { - throw IllegalArgumentException("No xsd were found from input dictionary!") - } + xsdMap = emptyMap() +// xsdMap = decodeInputToDictionary(dictionary, XSD_FOLDER) +// if (xsdMap.isEmpty()) { +// throw IllegalArgumentException("No xsd were found from input dictionary!") +// } } override fun create(settings: IPipelineCodecSettings?): IPipelineCodec { diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecSettings.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecSettings.kt index 18a35d6..ffe3fa8 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecSettings.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecSettings.kt @@ -19,6 +19,7 @@ import com.exactpro.th2.codec.api.IPipelineCodecSettings class XmlPipelineCodecSettings( val typePointer: String? = null, - val dirtyValidation: Boolean = false, - val expectsDeclaration: Boolean = true, + val encodeValidation: Boolean = false +// val dirtyValidation: Boolean = false, +// val expectsDeclaration: Boolean = true, ) : IPipelineCodecSettings \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/utils/Converter.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/utils/Converter.kt index 55c824f..3a5eaeb 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/utils/Converter.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/utils/Converter.kt @@ -27,17 +27,23 @@ import com.exactpro.th2.common.value.toValue import java.lang.IllegalArgumentException @Suppress("UNCHECKED_CAST") -private fun Map.toProtoValue(name: String = ""): Value { +private fun MutableMap.toProtoValue(name: String = ""): Value? { + this.removeSelfClosing() + if (this.isEmpty()) { + return null + } val message = message().also { builder -> builder.messageType = name } for ((key, value) in this) { message[key] = when (value) { - is Map<*, *> -> (value as Map).toProtoValue() + is Map<*, *> -> (value as MutableMap).toProtoValue() is String -> value - is ArrayList<*> -> when { - value[0] is Map<*, *> -> value.map { element -> (element as Map).toProtoValue() }.toValue() - else -> value.toValue() + is ArrayList<*> -> value.mapNotNull { + when (it) { + is Map<*, *> -> (it as MutableMap).toProtoValue() + else -> it.toValue() + } } null -> continue else -> error("Unsupported type of value: ${value::class.simpleName}") @@ -46,8 +52,9 @@ private fun Map.toProtoValue(name: String = ""): Value { return message.build().toValue() } -fun Map.toProto(type: String, rawMessage: RawMessage): Message { - val builder = toProtoValue(type).getMessage()?.toBuilder() ?: throw IllegalArgumentException("JsonNode $this does not contain a message") +fun MutableMap.toProto(type: String, rawMessage: RawMessage): Message { + val builder = toProtoValue(type)?.getMessage()?.toBuilder() + ?: throw IllegalArgumentException("JsonNode $this does not contain a message") val rawMetadata = rawMessage.metadata if (rawMessage.hasParentEventId()) builder.parentEventId = rawMessage.parentEventId @@ -60,4 +67,6 @@ fun Map.toProto(type: String, rawMessage: RawMessage): Message { } return builder.build() -} \ No newline at end of file +} + +fun MutableMap.removeSelfClosing() = this.apply { remove("-self-closing") } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XMLSchemaCore.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XMLSchemaCore.kt new file mode 100644 index 0000000..5fab0b9 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XMLSchemaCore.kt @@ -0,0 +1,98 @@ +package com.exactpro.th2.codec.xml.xsd + +import org.apache.ws.commons.schema.XmlSchema +import org.apache.ws.commons.schema.XmlSchemaAny +import org.apache.ws.commons.schema.XmlSchemaChoice +import org.apache.ws.commons.schema.XmlSchemaCollection +import org.apache.ws.commons.schema.XmlSchemaComplexType +import org.apache.ws.commons.schema.XmlSchemaElement +import org.apache.ws.commons.schema.XmlSchemaParticle +import org.apache.ws.commons.schema.XmlSchemaSequence +import org.apache.ws.commons.schema.utils.XmlSchemaObjectBase +import java.io.FileInputStream +import java.util.LinkedList +import java.util.Properties +import javax.xml.namespace.QName +import javax.xml.transform.stream.StreamSource +import kotlin.collections.ArrayList +import kotlin.collections.HashMap + +class XMLSchemaCore { + private val cachedURIXsds = LinkedList() // TODO: cache URI xsds here + val xsdProperties = Properties().also { it.load(Thread.currentThread().contextClassLoader.getResourceAsStream("xsds.properties")) } + + private val xsdElements: MutableMap> = HashMap() + + fun getXSDElements(xsdPath: String): Map> { + val xmlSchemaCollection = XmlSchemaCollection() + + // Schema contains the complete XSD content which needs to be parsed + val schema: XmlSchema = xmlSchemaCollection.read(StreamSource(FileInputStream(xsdPath))) + + schema.elements.forEach { + val element = XmlElementWrapper(it.value) + val qName = it.key + + xsdElements.putIfAbsent(qName, mutableListOf(element)) + + // Get all the elements based on the parent element + val childElement: XmlSchemaElement = xmlSchemaCollection.getElementByQName(qName) + + // Call method to get all the child elements + xsdElements.getChildElementNames(childElement) + } + + return xsdElements + } + + private fun MutableMap>.getChildElementNames(element: XmlSchemaElement) { + val elementType = element.schemaType + + if (elementType is XmlSchemaComplexType) { + val particle: XmlSchemaParticle? = elementType.particle + +// xsdElements.putIfAbsent(element.qName, mutableListOf(XmlElementWrapper(element))) + xsdElements.putIfAbsent(element.qName, mutableListOf()) + + if (particle is XmlSchemaSequence) { + particle.items.forEach { item -> + processItemElements(getItemElements(item), element) + } + } else if (particle is XmlSchemaChoice) { + particle.items.forEach { item -> + processItemElements(getItemElements(item), element) + } + } + } + } + + private fun MutableMap>.processItemElements(itemElements: Collection, + element: XmlSchemaElement) { + itemElements.forEach { + addChild(element.qName, XmlElementWrapper(it)) + // Call method recursively to get all subsequent element + getChildElementNames(it) + } + } + + private fun getItemElements(item: XmlSchemaObjectBase): Collection { + return when (item) { + is XmlSchemaElement -> listOf(item) + is XmlSchemaChoice -> item.items.mapNotNull { + if (it is XmlSchemaElement) { it } else { null } + } + is XmlSchemaSequence -> item.items.mapNotNull { + if (it is XmlSchemaElement) { it } else { null } + } + is XmlSchemaAny -> emptyList() + else -> { throw IllegalArgumentException("Not a valid type of $item") } + } + } + + private fun MutableMap>.addChild(qName: QName, child: XmlElementWrapper) { + val values: MutableList = this[qName] ?: ArrayList() + + values.add(child) + this[qName] = values + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XmlElementWrapper.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XmlElementWrapper.kt new file mode 100644 index 0000000..a66efef --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XmlElementWrapper.kt @@ -0,0 +1,43 @@ +package com.exactpro.th2.codec.xml.xsd + +import com.exactpro.th2.common.grpc.Value +import org.apache.ws.commons.schema.XmlSchemaComplexType +import org.apache.ws.commons.schema.XmlSchemaElement +import org.apache.ws.commons.schema.XmlSchemaType +import com.exactpro.th2.common.grpc.Value.KindCase.SIMPLE_VALUE +import com.exactpro.th2.common.grpc.Value.KindCase.MESSAGE_VALUE +import com.exactpro.th2.common.grpc.Value.KindCase.LIST_VALUE + +class XmlElementWrapper(element: XmlSchemaElement) { + private val type: XmlSchemaType? = element.schemaType + + val qName = element.qName ?: element.targetQName + + val elementType: Value.KindCase = when { + type == null && element.maxOccurs > 1 -> LIST_VALUE + type == null && element.maxOccurs == 1L -> MESSAGE_VALUE + type !is XmlSchemaComplexType -> SIMPLE_VALUE + element.maxOccurs > 1 -> LIST_VALUE + else -> MESSAGE_VALUE + } + + override fun toString() = "${qName.namespaceURI} - ${qName.localPart}" + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as XmlElementWrapper + + if (qName != other.qName) return false + if (elementType != other.elementType) return false + + return true + } + + override fun hashCode(): Int { + var result = qName?.hashCode() ?: 0 + result = 31 * result + elementType.hashCode() + return result + } +} \ No newline at end of file diff --git a/src/main/resources/xsds.properties b/src/main/resources/xsds.properties new file mode 100644 index 0000000..5efff57 --- /dev/null +++ b/src/main/resources/xsds.properties @@ -0,0 +1 @@ +www.w3.org/2000/09/xmldsig#=src/main/resources/xsds/2000_09_xmldsig#.xsd diff --git a/src/main/resources/xsds/2000_09_xmldsig#.xsd b/src/main/resources/xsds/2000_09_xmldsig#.xsd new file mode 100644 index 0000000..63c09a0 --- /dev/null +++ b/src/main/resources/xsds/2000_09_xmldsig#.xsd @@ -0,0 +1,253 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlCollectionTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlCollectionTest.kt index 7bd1fc5..f681ee6 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlCollectionTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlCollectionTest.kt @@ -21,6 +21,7 @@ import com.exactpro.th2.common.message.addField import com.exactpro.th2.common.message.addFields import com.exactpro.th2.common.message.message import org.junit.jupiter.api.Test +import java.util.* class XmlCollectionTest : XmlTest() { @@ -79,4 +80,40 @@ class XmlCollectionTest : XmlTest() { checkEncode(xml, msg) } + + @Test + fun `test decode array with self-closing tag in list element`() { + val xml = """ + + + + 1 + 2 + + + """.trimIndent() + val msg = parsedMessage("TestCollection").addFields( + "TestCollection", message().apply { + addField("array", listOf(message().apply { + addField("data", listOf("1", "2")) + })) + }, + ) + + checkDecode(xml, msg) + } + + @Test + fun `test decode array with self-closing tag as single element`() { + val xml = """ + + + + """.trimIndent() + val msg = parsedMessage("TestCollection").addFields( + "TestCollection", message(), + ) + + checkDecode(xml, msg) + } } \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlDecodeTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlDecodeTest.kt new file mode 100644 index 0000000..9df22ef --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlDecodeTest.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.codec.xml + +import com.exactpro.th2.codec.api.IPipelineCodec +import com.exactpro.th2.common.grpc.AnyMessage +import com.exactpro.th2.common.grpc.MessageGroup +import com.exactpro.th2.common.grpc.RawMessage +import com.google.protobuf.ByteString +import com.google.protobuf.util.JsonFormat +import org.junit.jupiter.api.Assertions.assertEquals +import java.io.ByteArrayInputStream +import kotlin.io.path.Path +import kotlin.io.path.readBytes + +class XmlDecodeTest { + + companion object { + private val XML_FILE_PATH = Path("tmp", "test.xml") + private val PROTO_FILE_PATH = Path("tmp", "test_orig.bin") + private const val ITERATIONS = 10_000 + private const val CYCLES = 5 + + @JvmStatic + fun main(args: Array) { + val xml = XML_FILE_PATH.readBytes() + val proto = PROTO_FILE_PATH.readBytes() + val message = AnyMessage.newBuilder().setRawMessage(RawMessage.newBuilder().apply { + metadataBuilder.apply { + protocol = XmlPipelineCodecFactory.PROTOCOL + idBuilder.connectionIdBuilder.sessionAlias = "test_session_alias" + } + body = ByteString.copyFrom(xml) + }) + val messageGroup = MessageGroup.newBuilder().apply { + addMessages(message) + }.build() + + val protoMessageGroup: MessageGroup = MessageGroup.parseFrom(proto) + + XmlPipelineCodecFactory().apply { + init(ByteArrayInputStream(byteArrayOf())) + }.use { factory -> + val codec = factory.create() + + for (cycle in 0 until CYCLES) { + parse(codec, messageGroup) + } + + parse(codec, messageGroup).also { + with(JsonFormat.printer()) { + assertEquals(print(protoMessageGroup), print(it)) + } + } + } + } + + private fun parse( + codec: IPipelineCodec, + messageGroup: MessageGroup + ): MessageGroup { + var millisecond = 0L + lateinit var result: MessageGroup + + for (i in 0..ITERATIONS) { + val start = System.currentTimeMillis() + result = codec.decode(messageGroup) + millisecond += System.currentTimeMillis() - start + } + println("Avg ${ITERATIONS.toDouble() / (millisecond) * 1_000}") + return result + } + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecTest.kt index 68b8d23..39812ae 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecTest.kt @@ -82,9 +82,8 @@ class XmlPipelineCodecTest : XmlTest() { } @Test - fun `test validation of xml declaration`() { - val withoutValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings(expectsDeclaration = false), mapOf()) - val withValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings(expectsDeclaration = true), mapOf()) + fun `test xml declaration`() { + val withoutValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings(), mapOf()) // Message with XML declaration var xml: MessageGroup = createMessageGroup(""" @@ -94,7 +93,6 @@ class XmlPipelineCodecTest : XmlTest() { """.trimIndent()) Assertions.assertDoesNotThrow { withoutValidationCodec.decode(xml) } - Assertions.assertDoesNotThrow { withValidationCodec.decode(xml) } // Formatted message with XML declaration xml = createMessageGroup(""" @@ -104,12 +102,11 @@ class XmlPipelineCodecTest : XmlTest() { """.trimIndent()) Assertions.assertDoesNotThrow { withoutValidationCodec.decode(xml) } - Assertions.assertThrows(IllegalStateException::class.java) { withValidationCodec.decode(xml) } } @Test - fun `test validation of xml with few root elements`() { - val withoutValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings(expectsDeclaration = false), mapOf()) + fun `test xml with few root elements`() { + val withoutValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings()) val xml = createMessageGroup(""" diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/XsdTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/XsdTest.kt index bce6e78..6eedfd6 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/xml/XsdTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/XsdTest.kt @@ -21,6 +21,7 @@ import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.MessageGroup import com.exactpro.th2.common.grpc.RawMessage import com.google.protobuf.ByteString +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import java.nio.file.Files import java.nio.file.Path @@ -44,6 +45,7 @@ class XsdTest : XmlTest() { } @Test + @Disabled("validation temporay disabled") fun `xsd not found exception`() { val xml = """ diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/utils/XmlTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/utils/XmlTest.kt index 6f9eef6..acb8b4c 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/xml/utils/XmlTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/utils/XmlTest.kt @@ -17,7 +17,6 @@ package com.exactpro.th2.codec.xml.utils import com.exactpro.th2.codec.api.IPipelineCodec import com.exactpro.th2.codec.xml.XmlPipelineCodec -import com.exactpro.th2.codec.xml.XmlPipelineCodecFactory.Companion.decodeInputToDictionary import com.exactpro.th2.codec.xml.XmlPipelineCodecSettings import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.Message @@ -26,13 +25,11 @@ import com.google.protobuf.TextFormat import mu.KotlinLogging import org.apache.commons.io.FileUtils import org.slf4j.Logger -import java.io.ByteArrayInputStream import java.io.File -import java.nio.file.Path -import java.util.Base64 +import java.util.* import kotlin.test.assertEquals -abstract class XmlTest(jsonPathToType: String? = null, nameOfXsdResource: String? = null) { +abstract class XmlTest(pathToType: String? = null) { protected val codec: IPipelineCodec @@ -58,12 +55,7 @@ abstract class XmlTest(jsonPathToType: String? = null, nameOfXsdResource: String } init { - val xsdMap = nameOfXsdResource?.run { - val zipBase64 = Thread.currentThread().contextClassLoader.getResource(nameOfXsdResource)!! - decodeInputToDictionary(ByteArrayInputStream(encodeFileToBase64Binary(zipBase64.file)), Path.of("tmp").toString()) - } ?: mapOf() - - codec = XmlPipelineCodec(XmlPipelineCodecSettings(jsonPathToType), xsdMap) + codec = XmlPipelineCodec(XmlPipelineCodecSettings(pathToType)) } protected fun encodeFileToBase64Binary(fileName: String): ByteArray { diff --git a/src/test/resources/log4j2.properties b/src/test/resources/log4j2.properties new file mode 100644 index 0000000..9a3b5b5 --- /dev/null +++ b/src/test/resources/log4j2.properties @@ -0,0 +1,22 @@ +name=Th2Logger + +# Console appender configuration +appender.console.type=Console +appender.console.name=consoleLogger +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n + +# Root logger level +rootLogger.level=INFO + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref=consoleLogger + +logger.th2.name=com.exactpro.th2 +logger.th2.level=INFO + +logger.evolution.name=com.exactpro.evolution +logger.evolution.level=INFO + +logger.cradle.name=com.exactpro.cradle +logger.cradle.level=INFO \ No newline at end of file