diff --git a/.github/workflows/build-dev-release.yml b/.github/workflows/build-dev-release.yml new file mode 100644 index 0000000..6cc5c82 --- /dev/null +++ b/.github/workflows/build-dev-release.yml @@ -0,0 +1,19 @@ +name: Build and publish dev release Docker image to Github Container Registry ghcr.io and publish dev release jar to sonatype + +on: workflow_dispatch + +jobs: + build: + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Sonatype,Docker' + devRelease: true + createTag: true + docker-username: ${{ github.actor }} + secrets: + docker-password: ${{ secrets.GITHUB_TOKEN }} + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} + nvd-api-key: ${{ secrets.NVD_APIKEY }} \ No newline at end of file diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml new file mode 100644 index 0000000..9251e4e --- /dev/null +++ b/.github/workflows/build-release.yml @@ -0,0 +1,19 @@ +name: Build and publish release Docker image to Github Container Registry ghcr.io and publish release jar to sonatype + +on: workflow_dispatch + +jobs: + build: + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Sonatype,Docker' + devRelease: false + createTag: true + docker-username: ${{ github.actor }} + secrets: + docker-password: ${{ secrets.GITHUB_TOKEN }} + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} + nvd-api-key: ${{ secrets.NVD_APIKEY }} \ No newline at end of file diff --git a/.github/workflows/build-sanpshot.yml b/.github/workflows/build-sanpshot.yml new file mode 100644 index 0000000..fc014bf --- /dev/null +++ b/.github/workflows/build-sanpshot.yml @@ -0,0 +1,24 @@ +name: Build and publish Docker image to Github Container Registry ghcr.io and publish snapshot jar to sonatype + +on: + push: + branches-ignore: + - master + - version-* + - dependabot** + paths-ignore: + - README.md + +jobs: + build-job: + uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main + with: + build-target: 'Sonatype,Docker' + docker-username: ${{ github.actor }} + secrets: + docker-password: ${{ secrets.GITHUB_TOKEN }} + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} + nvd-api-key: ${{ secrets.NVD_APIKEY }} \ No newline at end of file diff --git a/.github/workflows/ci-unwelcome-words.yml b/.github/workflows/ci-unwelcome-words.yml index cd7adcf..39d4010 100644 --- a/.github/workflows/ci-unwelcome-words.yml +++ b/.github/workflows/ci-unwelcome-words.yml @@ -5,19 +5,19 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.sha }} - - name: Checkout tool - uses: actions/checkout@v2 - with: - repository: exactpro-th2/ci-github-action - ref: master - token: ${{ secrets.PAT_CI_ACTION }} - path: ci-github-action - - name: Run CI action - uses: ./ci-github-action - with: - ref: ${{ github.sha }} + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + - name: Checkout tool + uses: actions/checkout@v4 + with: + repository: exactpro-th2/ci-github-action + ref: master + token: ${{ secrets.PAT_CI_ACTION }} + path: ci-github-action + - name: Run CI action + uses: ./ci-github-action + with: + ref: ${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/dev-docker-publish.yml b/.github/workflows/dev-docker-publish.yml deleted file mode 100644 index dbf999d..0000000 --- a/.github/workflows/dev-docker-publish.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Dev build and publish Docker distributions to Github Container Registry ghcr.io - -on: - push: - branches-ignore: - - master - - version-* - - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - # Prepare custom build version - - name: Get branch name - id: branch - run: echo ::set-output name=branch_name::${GITHUB_REF#refs/*/} - - name: Get release_version - id: ver - uses: christian-draeger/read-properties@1.0.1 - with: - path: gradle.properties - property: release_version - - name: Build custom release version - id: release_ver - run: echo ::set-output name=value::"${{ steps.ver.outputs.value }}-${{ steps.branch.outputs.branch_name }}-${{ github.run_id }}" - - name: Show custom release version - run: echo ${{ steps.release_ver.outputs.value }} - # Build and publish image - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} - - run: echo "::set-output name=REPOSITORY_NAME::$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" - id: meta - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${{ steps.release_ver.outputs.value }} - labels: com.exactpro.th2.${{ steps.meta.outputs.REPOSITORY_NAME }}=${{ steps.ver.outputs.value }} \ No newline at end of file diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index 9f51020..0000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build and publish Docker distributions to Github Container Registry ghcr.io - -on: - push: - branches: - - master - - version-* - paths: - - gradle.properties - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} - - run: echo "::set-output name=REPOSITORY_NAME::$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" - id: meta - - name: Read version from gradle.properties - id: read_property - uses: christian-draeger/read-properties@1.0.1 - with: - path: ./gradle.properties - property: release_version - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${{ steps.read_property.outputs.value }} - labels: com.exactpro.th2.${{ steps.meta.outputs.REPOSITORY_NAME }}=${{ steps.read_property.outputs.value }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c809ea4..432f34f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM gradle:6.8.2-jdk11 AS build +FROM gradle:8.7-jdk11 AS build ARG release_version COPY ./ . RUN gradle --no-daemon clean build dockerPrepare -Prelease_version=${release_version} diff --git a/README.md b/README.md index 525c117..3b3b9e0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Codec Xml via Xsd +# Codec Xml via Xsd (2.3.0) ![version](https://img.shields.io/badge/version-0.0.4-blue.svg) # How it works: @@ -58,12 +58,13 @@ Error from validation process can be disabled for test purposes by `dirtyValidat * typePointer - Path to message type value for decode (null by default) * dirtyValidation - Disable/enable error during validation phase. If disabled all errors will be only visible in log (false by default) -* expectsDeclaration - Disable/enable validation of declaration - is it exist or not (true by default) +* schemas - mapping between schema location value and a dictionary alias that should be used to load this schema ```yaml typePointer: /root/node/node2/type dirtyValidation: false -expectsDeclaration: false +schemas: + "test.xsd": "${dictionary_link:dict-xml}" ``` For example: @@ -77,7 +78,6 @@ spec: custom-config: codecSettings: typePointer: /root/node/node2/type - dirtyValidation: false ``` ## Required pins @@ -142,6 +142,30 @@ spec: ## Changelog +### v2.3.0 + +* Migrated to th2 gradle plugin `0.1.1` (bom: `4.6.1`) +* Updated common: `5.14.0-dev` +* Updated common-utils `2.2.3-dev` +* Updated codec `5.5.0-dev` +* Updated workflows + +### v2.2.0 + +* Migrate to StAX parser + +### v2.1.0 + +* th2 transport protocol support + +### v0.0.4 + +#### Changed: +* th2-common `3.44.0` +* th2-bom `4.1.0` +* kotlin `1.6.21` +* Dependency check pipeline step + ### v0.0.4 #### Feature: diff --git a/build.gradle b/build.gradle index f4f0184..e6a4673 100755 --- a/build.gradle +++ b/build.gradle @@ -1,133 +1,77 @@ -/* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - plugins { - id 'com.palantir.docker' version '0.25.0' - id 'org.jetbrains.kotlin.jvm' version "${kotlin_version}" - id 'com.github.harbby.gradle.serviceloader' version '1.1.5' - id 'application' -} - -ext { - sharedDir = file("${project.rootDir}/shared") - sailfishVersion = '3.2.1752' + id "org.jetbrains.kotlin.jvm" version "$kotlin_version" + id "org.jetbrains.kotlin.kapt" version "$kotlin_version" + id "application" + id "java-library" + id "maven-publish" + id "com.exactpro.th2.gradle.publish" version "0.1.1" + id "com.exactpro.th2.gradle.component" version "0.1.1" } group = 'com.exactpro.th2' version = release_version -sourceCompatibility = 11 -targetCompatibility = 11 - -ext { - junitVersion = '5.8.1' -} +kotlin.jvmToolchain(11) repositories { - mavenCentral() maven { name 'Sonatype_snapshots' url 'https://s01.oss.sonatype.org/content/repositories/snapshots/' } + + maven { + name 'Sonatype_snapshots' + url 'https://s01.oss.sonatype.org/content/repositories/snapshots/' + metadataSources { + mavenPom() + artifact() + } + } + maven { name 'Sonatype_releases' url 'https://s01.oss.sonatype.org/content/repositories/releases/' } - configurations.all { - resolutionStrategy.cacheChangingModulesFor 0, 'seconds' - resolutionStrategy.cacheDynamicVersionsFor 0, 'seconds' + maven { + name 'Sonatype_releases' + url 'https://s01.oss.sonatype.org/content/repositories/releases/' + metadataSources { + mavenPom() + artifact() + } } -} -jar { - manifest { - attributes( - 'Created-By': "${System.getProperty('java.version')} (${System.getProperty('java.vendor')})", - 'Specification-Title': '', - 'Specification-Vendor': 'Exactpro Systems LLC', - 'Implementation-Title': project.archivesBaseName, - 'Implementation-Vendor': 'Exactpro Systems LLC', - 'Implementation-Vendor-Id': 'com.exactpro', - 'Implementation-Version': project.version - ) + mavenCentral() + mavenLocal() + + configurations.configureEach { + resolutionStrategy.cacheChangingModulesFor 0, 'seconds' + resolutionStrategy.cacheDynamicVersionsFor 0, 'seconds' } } dependencies { - api platform('com.exactpro.th2:bom:3.1.0') - implementation 'com.exactpro.th2:common:3.31.5' - implementation 'com.exactpro.th2:codec:4.7.2' - implementation 'com.exactpro.th2:sailfish-utils:3.12.3' - - implementation "com.exactpro.sf:sailfish-core:${sailfishVersion}" - - implementation 'org.slf4j:slf4j-log4j12' - implementation 'org.slf4j:slf4j-api' - - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}" - implementation "org.jetbrains.kotlin:kotlin-reflect:${kotlin_version}" - - implementation 'com.fasterxml.jackson.core:jackson-core:2.13.1' - implementation 'com.github.javadev:underscore:1.69' - - testImplementation "org.jetbrains.kotlin:kotlin-test-junit5:${kotlin_version}" - testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}" - testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" - testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}" -} - -application { - mainClassName 'com.exactpro.th2.codec.MainKt' -} - -applicationName = 'service' - -dockerPrepare { - dependsOn distTar -} + implementation "com.exactpro.th2:common:5.14.0-dev" + implementation "com.exactpro.th2:common-utils:2.2.3-dev" + implementation "com.exactpro.th2:codec:5.5.0-dev" -docker { - copySpec.from(tarTree("$buildDir/distributions/${applicationName}.tar")) -} + implementation "org.slf4j:slf4j-api" + implementation "io.github.oshai:kotlin-logging:5.1.4" -sourceSets { - main.kotlin.srcDirs += "src/main/kotlin" -} - -compileKotlin { - kotlinOptions { - jvmTarget = "11" - } -} + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.github.javadev:underscore:1.104" + implementation "commons-io:commons-io" -compileTestKotlin { - kotlinOptions { - jvmTarget = "11" - } -} + implementation "org.apache.ws.xmlschema:xmlschema-core:2.3.1" -clean { - delete sharedDir -} + compileOnly "com.google.auto.service:auto-service:1.1.1" + annotationProcessor "com.google.auto.service:auto-service:1.1.1" + kapt "com.google.auto.service:auto-service:1.1.1" -serviceLoader { - serviceInterface 'com.exactpro.th2.codec.api.IPipelineCodecFactory' + testImplementation "org.jetbrains.kotlin:kotlin-test-junit5" } -test { - useJUnitPlatform() - //exclude 'com/exactpro/th2/codec/xml' -} +test.useJUnitPlatform() +application.mainClass = "com.exactpro.th2.codec.MainKt" +dependencyCheck.suppressionFile = file('suppressions.xml') \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 6552ee3..bd09e9b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ # -# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2023 Exactpro (Exactpro Systems Limited) # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,8 +13,8 @@ # limitations under the License. # -release_version=0.0.5 -kotlin_version=1.5.31 +release_version=2.3.0 +kotlin_version=1.8.22 description = 'th2 codec xml via xsd' vcs_url=https://github.com/th2-net/th2-codec-xml-via-xsd \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 0c9760e..cbb73a6 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ # -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2024 Exactpro (Exactpro Systems Limited) # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,8 +13,7 @@ # limitations under the License. # -#Thu Jul 02 11:31:27 GMT+04:00 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/NodeContent.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/NodeContent.kt new file mode 100644 index 0000000..8623fae --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/NodeContent.kt @@ -0,0 +1,165 @@ +/* + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.codec.xml + +import com.exactpro.th2.common.grpc.Value +import com.exactpro.th2.common.value.toValue +import javax.xml.namespace.QName + +typealias FieldName = String + +interface FieldAppender { + fun T.appendSimple(name: FieldName, value: String) + + fun T.appendNode(name: FieldName, node: NodeContent) + + fun T.appendNodeCollection(name: FieldName, nodes: List>) +} + +class NodeContent( + private val nodeName: QName, + decorator: XmlCodecStreamReader, + messageSupplier: () -> T, + private val appender: FieldAppender, +) { + private val messageBuilder: T by lazy(messageSupplier) + private val textSB = StringBuilder() + + private val childNodes: MutableMap>> = mutableMapOf() + var isMessage: Boolean = false + private set + private val isEmpty: Boolean + get() = !isMessage && textSB.isEmpty() + + val name: String = nodeName.toNodeName() + + init { + decorator.namespaceCount.let { size -> + if (size > 0) { + for (i in 0 until size) { + decorator.getNamespaceURI(i).also { value -> + val prefix = decorator.namespaceContext.getPrefix(value) + + with(appender) { + messageBuilder.appendSimple(makeFieldName(NAMESPACE, prefix, true), value) + } + } + } + isMessage = true + } + } + decorator.attributeCount.let { size -> + if (size > 0) { + for (i in 0 until size) { + val localPart = decorator.getAttributeLocalName(i) + val prefix = decorator.getAttributePrefix(i) + + with(appender) { + messageBuilder.appendSimple(makeFieldName(prefix, localPart, true), decorator.getAttributeValue(i)) + } + } + isMessage = true + } + } + } + + fun putChild(name: QName, node: NodeContent) { + childNodes.compute(name) { _, value -> + value + ?.apply { add(node) } + ?: mutableListOf(node) + } + isMessage = true + } + + fun appendText(text: String) { + if (text.isNotBlank()) { + textSB.append(text) + } + } + + fun release() { + if (isMessage) { + childNodes.forEach { (name, values) -> + try { + val notEmptyValues = values.filterNot(NodeContent<*>::isEmpty) + + if (notEmptyValues.isNotEmpty()) { + val first = notEmptyValues.first() + with(appender) { + when (values.size) { // Clarify type of element: list or single + 0 -> error("Sub element $name hasn't got values") + 1 -> messageBuilder.appendNode(first.name, first) + else -> messageBuilder.appendNodeCollection( + first.name, + notEmptyValues, + ) + } + } + } + } catch (e: RuntimeException) { + throw IllegalStateException("The `$name` field can't be released in the `$nodeName` node", e) + } + } + if(textSB.isNotBlank()) { + with(appender) { + messageBuilder.appendSimple(TEXT_FIELD_NAME, textSB.toString()) + } + } + } + } + + fun toMessage(): T { + check(isMessage) { + "The $nodeName node isn't message" + } + return messageBuilder + } + + fun toText(): String { + check(!isMessage) { + "The $nodeName is a message" + } + return textSB.toString() + } + + override fun toString(): String { + return "NodeContent(nodeName=$nodeName, childNodes=$childNodes, text=$textSB)" + } + + private fun Sequence.toListValue(): Value = Value.newBuilder().apply { + listValueBuilder.apply { + forEach(::addValues) + } + }.build() + + private fun toValue(): Value = if (isMessage) { + messageBuilder.toValue() + } else { + textSB.toValue() + } + + companion object { + private const val NAMESPACE = "xmlns" + private const val TEXT_FIELD_NAME = "#text" + + private fun QName.toNodeName(): String = makeFieldName(prefix, localPart) + + private fun makeFieldName(first: String, second: String, isAttribute: Boolean = false): String { + return "${if (isAttribute) "-" else ""}$first${if (first.isNotBlank() && second.isNotBlank()) ":" else ""}$second" + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/TransportFieldAppender.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/TransportFieldAppender.kt new file mode 100644 index 0000000..0a24fc4 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/TransportFieldAppender.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.codec.xml + +object TransportFieldAppender : FieldAppender> { + override fun MutableMap.appendSimple(name: FieldName, value: String) { + put(name, value) + } + + override fun MutableMap.appendNode(name: FieldName, node: NodeContent>) { + put(name, node.extractValue()) + } + + override fun MutableMap.appendNodeCollection( + name: FieldName, + nodes: List>>, + ) { + put(name, nodes.asSequence().map { it.extractValue() }.toList()) + } + + private fun NodeContent>.extractValue(): Any = + if (isMessage) { + toMessage() + } else { + toText() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlCodecStreamReader.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlCodecStreamReader.kt new file mode 100644 index 0000000..416b9b4 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlCodecStreamReader.kt @@ -0,0 +1,80 @@ +/* + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.codec.xml + +import java.io.ByteArrayInputStream +import java.util.ArrayDeque +import javax.xml.stream.XMLInputFactory +import javax.xml.stream.XMLStreamException +import javax.xml.stream.util.StreamReaderDelegate + +class XmlCodecStreamReader( + body: ByteArray, + private val messageSupplier: () -> T, + private val appender: FieldAppender, +) + : StreamReaderDelegate( + XML_INPUT_FACTORY.createXMLStreamReader( + ByteArrayInputStream(body) + ) +), AutoCloseable { + private lateinit var rootNode: NodeContent + private lateinit var messageType: String + + private val elements = ArrayDeque>() + + @Throws(XMLStreamException::class) + override fun next(): Int = super.next().also { eventCode -> + when (eventCode) { + START_ELEMENT -> { + val qName = name + NodeContent(qName, this, messageSupplier, appender).also { nodeContent -> + if (elements.isNotEmpty()) { + elements.peek() + .putChild(qName, nodeContent) + } + + elements.push(nodeContent) + + if (!this::messageType.isInitialized) { + messageType = localName + } + if (!this::rootNode.isInitialized) { + rootNode = nodeContent + } + } + } + CHARACTERS -> elements.peek().appendText(text) + END_ELEMENT -> elements.pop().also(NodeContent<*>::release) + } + } + + fun getMessage(): T { + check(elements.isEmpty()) { + "Some of XML nodes aren't closed ${elements.joinToString { it.name }}" + } + + return messageSupplier().apply { + with(appender) { + appendNode(rootNode.name, rootNode) + } + } + } + + companion object { + private val XML_INPUT_FACTORY = XMLInputFactory.newInstance() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodec.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodec.kt index baf975a..2ca97fd 100755 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodec.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodec.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -17,145 +17,201 @@ package com.exactpro.th2.codec.xml import com.exactpro.th2.codec.DecodeException import com.exactpro.th2.codec.api.IPipelineCodec +import com.exactpro.th2.codec.api.IReportingContext import com.exactpro.th2.codec.xml.utils.toMap import com.exactpro.th2.codec.xml.utils.toProto +import com.exactpro.th2.codec.xml.utils.toTransport import com.exactpro.th2.codec.xml.xsd.XsdValidator -import com.exactpro.th2.common.grpc.AnyMessage -import com.exactpro.th2.common.grpc.Message -import com.exactpro.th2.common.grpc.MessageGroup -import com.exactpro.th2.common.grpc.RawMessage -import com.exactpro.th2.common.message.logId import com.exactpro.th2.common.message.messageType import com.exactpro.th2.common.message.toJson -import com.github.underscore.lodash.Xml -import com.google.protobuf.ByteString -import org.slf4j.Logger -import org.slf4j.LoggerFactory +import com.exactpro.th2.common.message.logId +import com.exactpro.th2.common.grpc.AnyMessage as ProtoAnyMessage +import com.exactpro.th2.common.grpc.Message as ProtoMessage +import com.exactpro.th2.common.grpc.MessageGroup as ProtoMessageGroup +import com.exactpro.th2.common.grpc.RawMessage as ProtoRawMessage +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageGroup +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.RawMessage +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.toByteArray +import com.exactpro.th2.common.utils.message.transport.getField +import com.github.underscore.Xml +import com.google.protobuf.UnsafeByteOperations +import io.netty.buffer.Unpooled +import io.github.oshai.kotlinlogging.KotlinLogging +import java.io.InputStream import java.nio.charset.Charset -import java.nio.file.Path - -open class XmlPipelineCodec(private val settings: XmlPipelineCodecSettings, xsdMap: Map) : IPipelineCodec { +import java.util.Locale - private val pointer = settings.typePointer?.split("/")?.filterNot { it.isBlank() } +open class XmlPipelineCodec(settings: XmlPipelineCodecSettings, xsdMap: Map InputStream> = emptyMap()) : IPipelineCodec { + private val pointer: List = settings.typePointer + ?.split("/")?.filter(String::isNotBlank) + ?: listOf() private var xmlCharset: Charset = Charsets.UTF_8 private val validator = XsdValidator(xsdMap, settings.dirtyValidation) - override fun encode(messageGroup: MessageGroup): MessageGroup { + override fun encode(messageGroup: ProtoMessageGroup, context: IReportingContext): ProtoMessageGroup { val messages = messageGroup.messagesList if (messages.none { it.hasMessage() }) { return messageGroup } - return MessageGroup.newBuilder().addAllMessages( + return ProtoMessageGroup.newBuilder().addAllMessages( messages.map { anyMsg -> if (anyMsg.hasMessage() && checkProtocol(anyMsg.message.metadata.protocol)) - AnyMessage.newBuilder().setRawMessage(encodeOne(anyMsg.message)).build() + ProtoAnyMessage.newBuilder().setRawMessage(encodeOne(anyMsg.message)).build() else anyMsg } ).build() } + override fun encode(messageGroup: MessageGroup, context: IReportingContext): MessageGroup { + val messages = messageGroup.messages + if (messages.none { it is ParsedMessage }) { + return messageGroup + } + + return MessageGroup( + messages.map { anyMsg -> + if (anyMsg is ParsedMessage && checkProtocol(anyMsg.protocol)) + encodeOne(anyMsg) + else anyMsg + } + ) + } + //FIXME: move this check into the codec-core project private fun checkProtocol(msgProtocol: String?): Boolean { return msgProtocol.isNullOrEmpty() || msgProtocol == XmlPipelineCodecFactory.PROTOCOL } - private fun encodeOne(message: Message): RawMessage { - + private fun encodeOne(message: ProtoMessage): ProtoRawMessage { val map = message.toMap() - val xmlString = Xml.toXml(map) + val xmlStringBuffer = Xml.toXml(map).toByteArray(xmlCharset) - validator.validate(xmlString.toByteArray()) - LOGGER.debug("Validation of incoming parsed message complete: ${message.messageType}") + validator.validate(xmlStringBuffer) + LOGGER.debug { "Validation of incoming parsed message complete: ${message.messageType}" } - return RawMessage.newBuilder().apply { + return ProtoRawMessage.newBuilder().apply { if (message.hasParentEventId()) parentEventId = message.parentEventId metadataBuilder.putAllProperties(message.metadata.propertiesMap) metadataBuilder.protocol = XmlPipelineCodecFactory.PROTOCOL metadataBuilder.id = message.metadata.id - metadataBuilder.timestamp = message.metadata.timestamp - body = ByteString.copyFrom(xmlString, xmlCharset) + body = UnsafeByteOperations.unsafeWrap(xmlStringBuffer) }.build() } + private fun encodeOne(message: ParsedMessage): RawMessage { + val map = message.body + val xmlStringBuffer = Xml.toXml(map).toByteArray(xmlCharset) + + validator.validate(xmlStringBuffer) + LOGGER.debug { "Validation of incoming parsed message complete: ${message.type}" } + + return RawMessage( + id = message.id, + eventId = message.eventId, + metadata = message.metadata, + protocol = XmlPipelineCodecFactory.PROTOCOL, + body = Unpooled.wrappedBuffer(xmlStringBuffer) + ) + } - override fun decode(messageGroup: MessageGroup): MessageGroup { + override fun decode(messageGroup: ProtoMessageGroup, context: IReportingContext): ProtoMessageGroup { val messages = messageGroup.messagesList if (messages.none { it.hasRawMessage() }) { return messageGroup } - return MessageGroup.newBuilder().apply { + return ProtoMessageGroup.newBuilder().apply { messages.forEach { input -> - if (input.hasRawMessage() && checkProtocol(input.rawMessage.metadata.protocol)) + if (input.hasRawMessage() && checkProtocol(input.rawMessage.metadata.protocol)) { try { - addMessages(AnyMessage.newBuilder().setMessage(decodeOne(input.rawMessage)).build()) + addMessages(ProtoAnyMessage.newBuilder().setMessage(decodeOneProto(input.rawMessage)).build()) } catch (e: Exception) { throw IllegalStateException("Can not decode message = ${input.rawMessage.toJson()}", e) } - else { + } else { addMessages(input) } } }.build() } - private fun decodeOne(rawMessage: RawMessage): Message { - try { - validator.validate(rawMessage.body.toByteArray()) - LOGGER.debug("Validation of incoming raw message complete: ${rawMessage.logId}") - val xmlString = rawMessage.body.toStringUtf8() - @Suppress("UNCHECKED_CAST") - val map = Xml.fromXml(xmlString) as MutableMap - - LOGGER.trace("Result of the 'Xml.fromXml' method is ${map.keys} for $xmlString") - map -= STANDALONE - map -= ENCODING - - if (OMIT_XML_DECLARATION in map) { - // U library will tell by this option is there no declaration - check(!settings.expectsDeclaration || map[OMIT_XML_DECLARATION] == NO) { "Expecting declaration inside xml data" } - map -= OMIT_XML_DECLARATION - } + override fun decode(messageGroup: MessageGroup, context: IReportingContext): MessageGroup { + val messages = messageGroup.messages + if (messages.none { it is RawMessage }) { + return messageGroup + } - if (map.size > 1) { - error("There was more than one root node in processed xml, result json has [${map.size}]: ${map.keys.joinToString(", ")}") + return MessageGroup( + messages.map { input -> + if (input is RawMessage && checkProtocol(input.protocol)) { + try { + decodeOneTransport(input) + } catch (e: Exception) { + throw IllegalStateException("Can not decode message = $input", e) + } + } else { + input + } } + ) + } - val msgType: String = pointer?.let { map.getNode(it) } ?: map.keys.first() + private fun decodeOne(body: ByteArray, xmlString: String, logId: String): Pair> { + try { + validator.validate(body) + + XmlCodecStreamReader(body, { linkedMapOf() }, TransportFieldAppender).use { reader -> + while (reader.hasNext()) { + reader.next() + } + val message: MutableMap = reader.getMessage() + if (message.size > 1) { + error( + "There was more than one root node in processed xml, result json has [${message.size}]: " + + message.keys.joinToString(", ") + ) + } + val msgType: String = if (pointer.isEmpty()) { + message.keys.first() // first tag name + } else { + checkNotNull(message.getField(*pointer.toTypedArray())?.toString()) { + "message type at $pointer is null in message $message" + } + } + return msgType to message + } + } catch (e: Exception) { + throw DecodeException("Can not decode message ${logId}. Can not parse XML. $xmlString", e) + } + } + private fun decodeOneProto(rawMessage: ProtoRawMessage): ProtoMessage { + val xmlString = rawMessage.body.toString(Charsets.UTF_8) + try { + val (msgType, map) = decodeOne(rawMessage.body.toByteArray(), xmlString, rawMessage.logId) return map.toProto(msgType, rawMessage) } catch (e: Exception) { - throw DecodeException("Can not decode message. Can not parse XML. ${rawMessage.body.toStringUtf8()}", e) + throw DecodeException("Can not decode message. Can not parse XML. $xmlString", e) } } - private inline fun Map<*,*>.getNode(pointer: List): T { - var current: Any = this - for (name in pointer) { - current = (current as? Map<*, *>)?.get(name) ?: error("Can not find element by name '$name' in path: $pointer") + private fun decodeOneTransport(rawMessage: RawMessage): ParsedMessage { + val xmlString = rawMessage.body.toString(Charsets.UTF_8) + try { + val (msgType, map) = decodeOne(rawMessage.body.toByteArray(), xmlString, rawMessage.logId) + return map.toTransport(msgType, rawMessage) + } catch (e: Exception) { + throw DecodeException("Can not decode message. Can not parse XML. $xmlString", e) } - return current as T } + private val RawMessage.logId: String + get() = "${id.sessionAlias}:${id.direction.toString().lowercase(Locale.getDefault())}:${id.sequence}${id.subsequence.joinToString("") { ".$it" }}" companion object { - private val LOGGER: Logger = LoggerFactory.getLogger(XmlPipelineCodec::class.java) - - private const val NO = "no" - - /** - * The constant from [Xml.OMITXMLDECLARATION] - */ - private const val OMIT_XML_DECLARATION = "#omit-xml-declaration" - /** - * The constant from [Xml.ENCODING] - */ - private const val ENCODING = "#encoding" - /** - * The constant from [Xml.STANDALONE] - */ - private const val STANDALONE = "#standalone" + private val LOGGER = KotlinLogging.logger {} } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecFactory.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecFactory.kt index f6f000f..a81eaf2 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecFactory.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecFactory.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -16,50 +16,52 @@ package com.exactpro.th2.codec.xml import com.exactpro.th2.codec.api.IPipelineCodec +import com.exactpro.th2.codec.api.IPipelineCodecContext import com.exactpro.th2.codec.api.IPipelineCodecFactory import com.exactpro.th2.codec.api.IPipelineCodecSettings -import com.exactpro.th2.codec.xml.utils.ZipBase64Codec -import mu.KotlinLogging -import org.apache.commons.io.FileUtils +import io.github.oshai.kotlinlogging.KotlinLogging import java.io.InputStream -import java.nio.file.Files -import java.nio.file.Path +import com.google.auto.service.AutoService +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock +@AutoService(IPipelineCodecFactory::class) class XmlPipelineCodecFactory : IPipelineCodecFactory { override val settingsClass: Class = XmlPipelineCodecSettings::class.java - override val protocol: String = PROTOCOL - lateinit var xsdMap: Map + override val protocols: Set + get() = PROTOCOLS + private lateinit var context: IPipelineCodecContext + private val lock = ReentrantLock() + @Volatile + private lateinit var xsdMap: Map InputStream> - override fun init(dictionary: InputStream) { - xsdMap = decodeInputToDictionary(dictionary, XSD_FOLDER) - if (xsdMap.isEmpty()) { - throw IllegalArgumentException("No xsd were found from input dictionary!") - } + override fun init(pipelineCodecContext: IPipelineCodecContext) { + context = pipelineCodecContext } override fun create(settings: IPipelineCodecSettings?): IPipelineCodec { - return XmlPipelineCodec(settings as? XmlPipelineCodecSettings ?: XmlPipelineCodecSettings(), xsdMap) + val codecSettings = settings as? XmlPipelineCodecSettings ?: XmlPipelineCodecSettings() + return XmlPipelineCodec(codecSettings, initXsdMap(codecSettings)) + } + + private fun initXsdMap(settings: XmlPipelineCodecSettings): Map InputStream> { + return lock.withLock { + if (::xsdMap.isInitialized) { + xsdMap + } else { + LOGGER.info { "Loading schemas from settings" } + settings.schemas.mapValues { (_, alias) -> + { context[alias] } + }.also { + xsdMap = it + } + } + } } companion object { - private const val XSD_FOLDER: String = "/tmp/xsd" private val LOGGER = KotlinLogging.logger { } const val PROTOCOL = "XML" - - fun decodeInputToDictionary(dictionary: InputStream, parentDir: String): Map = dictionary.use { - val parentDirPath = Path.of(parentDir) - Files.createDirectory(parentDirPath) - val xsdDir = Files.createTempDirectory(parentDirPath, "") - val pathMap = ZipBase64Codec.decode(it.readAllBytes(), xsdDir.toFile()) - - LOGGER.info { - "Decoded xsd files: ${ - FileUtils.listFiles(parentDirPath.toFile(), Array(1) {"proto"}, true).map { file -> - parentDirPath.relativize(file.toPath()) - }.toList() - }" - } - pathMap - } + private val PROTOCOLS = setOf(PROTOCOL) } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecSettings.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecSettings.kt index 18a35d6..8bb4754 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecSettings.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecSettings.kt @@ -15,10 +15,11 @@ package com.exactpro.th2.codec.xml +import com.exactpro.th2.codec.api.DictionaryAlias import com.exactpro.th2.codec.api.IPipelineCodecSettings class XmlPipelineCodecSettings( val typePointer: String? = null, val dirtyValidation: Boolean = false, - val expectsDeclaration: Boolean = true, + val schemas: Map = emptyMap(), ) : IPipelineCodecSettings \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/utils/Converter.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/utils/Converter.kt index 3a5eaeb..7318d32 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/utils/Converter.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/utils/Converter.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -17,20 +17,23 @@ package com.exactpro.th2.codec.xml.utils import com.exactpro.th2.codec.xml.XmlPipelineCodecFactory import com.exactpro.th2.common.grpc.Message -import com.exactpro.th2.common.grpc.RawMessage +import com.exactpro.th2.common.grpc.Message as ProtoMessage +import com.exactpro.th2.common.grpc.RawMessage as ProtoRawMessage import com.exactpro.th2.common.grpc.Value import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.messageType import com.exactpro.th2.common.message.set +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.RawMessage import com.exactpro.th2.common.value.getMessage import com.exactpro.th2.common.value.toValue import java.lang.IllegalArgumentException @Suppress("UNCHECKED_CAST") -private fun MutableMap.toProtoValue(name: String = ""): Value? { +private fun MutableMap.toProtoValue(name: String = ""): Value { this.removeSelfClosing() if (this.isEmpty()) { - return null + return Message.getDefaultInstance().toValue() } val message = message().also { builder -> builder.messageType = name @@ -39,7 +42,7 @@ private fun MutableMap.toProtoValue(name: String = ""): Value? { message[key] = when (value) { is Map<*, *> -> (value as MutableMap).toProtoValue() is String -> value - is ArrayList<*> -> value.mapNotNull { + is List<*> -> value.mapNotNull { when (it) { is Map<*, *> -> (it as MutableMap).toProtoValue() else -> it.toValue() @@ -52,8 +55,8 @@ private fun MutableMap.toProtoValue(name: String = ""): Value? { return message.build().toValue() } -fun MutableMap.toProto(type: String, rawMessage: RawMessage): Message { - val builder = toProtoValue(type)?.getMessage()?.toBuilder() +fun MutableMap.toProto(type: String, rawMessage: ProtoRawMessage): ProtoMessage { + val builder = toProtoValue(type).getMessage()?.toBuilder() ?: throw IllegalArgumentException("JsonNode $this does not contain a message") val rawMetadata = rawMessage.metadata @@ -61,7 +64,6 @@ fun MutableMap.toProto(type: String, rawMessage: RawMessage): Message builder.metadataBuilder.apply { id = rawMetadata.id - timestamp = rawMetadata.timestamp protocol = XmlPipelineCodecFactory.PROTOCOL putAllProperties(rawMetadata.propertiesMap) } @@ -69,4 +71,15 @@ fun MutableMap.toProto(type: String, rawMessage: RawMessage): Message return builder.build() } +fun Map.toTransport(type: String, rawMessage: RawMessage): ParsedMessage { + return ParsedMessage( + id = rawMessage.id, + eventId = rawMessage.eventId, + type = type, + metadata = rawMessage.metadata, + protocol = XmlPipelineCodecFactory.PROTOCOL, + body = this + ) +} + fun MutableMap.removeSelfClosing() = this.apply { remove("-self-closing") } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XMLSchemaCore.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XMLSchemaCore.kt new file mode 100644 index 0000000..5fab0b9 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XMLSchemaCore.kt @@ -0,0 +1,98 @@ +package com.exactpro.th2.codec.xml.xsd + +import org.apache.ws.commons.schema.XmlSchema +import org.apache.ws.commons.schema.XmlSchemaAny +import org.apache.ws.commons.schema.XmlSchemaChoice +import org.apache.ws.commons.schema.XmlSchemaCollection +import org.apache.ws.commons.schema.XmlSchemaComplexType +import org.apache.ws.commons.schema.XmlSchemaElement +import org.apache.ws.commons.schema.XmlSchemaParticle +import org.apache.ws.commons.schema.XmlSchemaSequence +import org.apache.ws.commons.schema.utils.XmlSchemaObjectBase +import java.io.FileInputStream +import java.util.LinkedList +import java.util.Properties +import javax.xml.namespace.QName +import javax.xml.transform.stream.StreamSource +import kotlin.collections.ArrayList +import kotlin.collections.HashMap + +class XMLSchemaCore { + private val cachedURIXsds = LinkedList() // TODO: cache URI xsds here + val xsdProperties = Properties().also { it.load(Thread.currentThread().contextClassLoader.getResourceAsStream("xsds.properties")) } + + private val xsdElements: MutableMap> = HashMap() + + fun getXSDElements(xsdPath: String): Map> { + val xmlSchemaCollection = XmlSchemaCollection() + + // Schema contains the complete XSD content which needs to be parsed + val schema: XmlSchema = xmlSchemaCollection.read(StreamSource(FileInputStream(xsdPath))) + + schema.elements.forEach { + val element = XmlElementWrapper(it.value) + val qName = it.key + + xsdElements.putIfAbsent(qName, mutableListOf(element)) + + // Get all the elements based on the parent element + val childElement: XmlSchemaElement = xmlSchemaCollection.getElementByQName(qName) + + // Call method to get all the child elements + xsdElements.getChildElementNames(childElement) + } + + return xsdElements + } + + private fun MutableMap>.getChildElementNames(element: XmlSchemaElement) { + val elementType = element.schemaType + + if (elementType is XmlSchemaComplexType) { + val particle: XmlSchemaParticle? = elementType.particle + +// xsdElements.putIfAbsent(element.qName, mutableListOf(XmlElementWrapper(element))) + xsdElements.putIfAbsent(element.qName, mutableListOf()) + + if (particle is XmlSchemaSequence) { + particle.items.forEach { item -> + processItemElements(getItemElements(item), element) + } + } else if (particle is XmlSchemaChoice) { + particle.items.forEach { item -> + processItemElements(getItemElements(item), element) + } + } + } + } + + private fun MutableMap>.processItemElements(itemElements: Collection, + element: XmlSchemaElement) { + itemElements.forEach { + addChild(element.qName, XmlElementWrapper(it)) + // Call method recursively to get all subsequent element + getChildElementNames(it) + } + } + + private fun getItemElements(item: XmlSchemaObjectBase): Collection { + return when (item) { + is XmlSchemaElement -> listOf(item) + is XmlSchemaChoice -> item.items.mapNotNull { + if (it is XmlSchemaElement) { it } else { null } + } + is XmlSchemaSequence -> item.items.mapNotNull { + if (it is XmlSchemaElement) { it } else { null } + } + is XmlSchemaAny -> emptyList() + else -> { throw IllegalArgumentException("Not a valid type of $item") } + } + } + + private fun MutableMap>.addChild(qName: QName, child: XmlElementWrapper) { + val values: MutableList = this[qName] ?: ArrayList() + + values.add(child) + this[qName] = values + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XmlElementWrapper.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XmlElementWrapper.kt new file mode 100644 index 0000000..a66efef --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XmlElementWrapper.kt @@ -0,0 +1,43 @@ +package com.exactpro.th2.codec.xml.xsd + +import com.exactpro.th2.common.grpc.Value +import org.apache.ws.commons.schema.XmlSchemaComplexType +import org.apache.ws.commons.schema.XmlSchemaElement +import org.apache.ws.commons.schema.XmlSchemaType +import com.exactpro.th2.common.grpc.Value.KindCase.SIMPLE_VALUE +import com.exactpro.th2.common.grpc.Value.KindCase.MESSAGE_VALUE +import com.exactpro.th2.common.grpc.Value.KindCase.LIST_VALUE + +class XmlElementWrapper(element: XmlSchemaElement) { + private val type: XmlSchemaType? = element.schemaType + + val qName = element.qName ?: element.targetQName + + val elementType: Value.KindCase = when { + type == null && element.maxOccurs > 1 -> LIST_VALUE + type == null && element.maxOccurs == 1L -> MESSAGE_VALUE + type !is XmlSchemaComplexType -> SIMPLE_VALUE + element.maxOccurs > 1 -> LIST_VALUE + else -> MESSAGE_VALUE + } + + override fun toString() = "${qName.namespaceURI} - ${qName.localPart}" + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as XmlElementWrapper + + if (qName != other.qName) return false + if (elementType != other.elementType) return false + + return true + } + + override fun hashCode(): Int { + var result = qName?.hashCode() ?: 0 + result = 31 * result + elementType.hashCode() + return result + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XsdErrorHandler.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XsdErrorHandler.kt index 70d9643..ea99c0a 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XsdErrorHandler.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XsdErrorHandler.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -15,8 +15,7 @@ package com.exactpro.th2.codec.xml.xsd -import mu.KotlinLogging -import org.slf4j.Logger +import io.github.oshai.kotlinlogging.KotlinLogging import org.xml.sax.ErrorHandler import org.xml.sax.SAXException import org.xml.sax.SAXParseException @@ -44,11 +43,11 @@ class XsdErrorHandler : ErrorHandler { val lineNumber: Int = exception.lineNumber val columnNumber: Int = exception.columnNumber val message: String? = exception.message - LOGGER.error("[$level] line nr: $lineNumber column nr: $columnNumber \nmessage: $message") + LOGGER.error { "[$level] line nr: $lineNumber column nr: $columnNumber \nmessage: $message" } throw SAXException("[$level] line nr: $lineNumber column nr: $columnNumber \nmessage: $message", exception) } companion object { - private val LOGGER: Logger = KotlinLogging.logger { } + private val LOGGER = KotlinLogging.logger { } } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XsdValidator.kt b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XsdValidator.kt index 5d31285..1a3d3e0 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XsdValidator.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/xml/xsd/XsdValidator.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -16,23 +16,23 @@ package com.exactpro.th2.codec.xml.xsd import com.exactpro.th2.codec.CodecException -import mu.KotlinLogging -import org.slf4j.Logger +import io.github.oshai.kotlinlogging.KotlinLogging import org.w3c.dom.Document import org.w3c.dom.NamedNodeMap import org.w3c.dom.Node import org.w3c.dom.NodeList import java.io.ByteArrayInputStream -import java.nio.file.Path +import java.io.InputStream import javax.xml.XMLConstants import javax.xml.parsers.DocumentBuilder import javax.xml.parsers.DocumentBuilderFactory import javax.xml.parsers.ParserConfigurationException import javax.xml.transform.dom.DOMSource +import javax.xml.transform.stream.StreamSource import javax.xml.validation.SchemaFactory import javax.xml.validation.Validator -class XsdValidator(private val xsdMap: Map, private val dirtyValidation: Boolean) { +class XsdValidator(private val xsdMap: Map InputStream>, private val dirtyValidation: Boolean) { fun validate(xml: ByteArray) { try { @@ -44,10 +44,9 @@ class XsdValidator(private val xsdMap: Map, private val dirtyValid attributes.forEach { attribute -> val schemas = getSchemas(attribute.nodeValue) schemas.forEach { schema -> - val xsdPath = checkNotNull(xsdMap[schema.value]) { "Cannot find xsd for current `${attribute.nodeName}` attribute: ${attribute.nodeValue}" } + val xsdLoader = checkNotNull(xsdMap[schema.value]) { "Cannot find xsd for current `${attribute.nodeName}` attribute: ${attribute.nodeValue}" } - val xsdFile = xsdPath.toFile() - val schemaFile = SCHEMA_FACTORY.newSchema(xsdFile) // is it worth for each time?? + val schemaFile = SCHEMA_FACTORY.newSchema(StreamSource(xsdLoader())) // is it worth for each time?? val validator: Validator = schemaFile.newValidator().apply { errorHandler = XsdErrorHandler() @@ -55,18 +54,17 @@ class XsdValidator(private val xsdMap: Map, private val dirtyValid val item = documentXML.getElementsByTagNameNS(attribute.nodeValue, "*").item(0) validator.validate(DOMSource(item)) - LOGGER.debug("Validation of raw message with XSD: ${xsdPath.fileName} finished") + LOGGER.debug { "Validation of raw message with XSD: ${schema.value} finished" } } } } } catch (e: Exception) { if (dirtyValidation) { - LOGGER.warn("VALIDATION ERROR: ", e) + LOGGER.warn(e) { "VALIDATION ERROR: " } } else { throw e } } - } private fun getSchemas(input: String): Map = mutableMapOf().apply { @@ -101,10 +99,9 @@ class XsdValidator(private val xsdMap: Map, private val dirtyValid } } - companion object { private const val SCHEMA_NAME_PROPERTY = "schemaLocation" - private val LOGGER: Logger = KotlinLogging.logger { } + private val LOGGER = KotlinLogging.logger { } private val SCHEMA_FACTORY = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI).apply { errorHandler = XsdErrorHandler() @@ -140,5 +137,4 @@ class XsdValidator(private val xsdMap: Map, private val dirtyValid } } } - } \ No newline at end of file diff --git a/src/main/resources/xsds.properties b/src/main/resources/xsds.properties new file mode 100644 index 0000000..5efff57 --- /dev/null +++ b/src/main/resources/xsds.properties @@ -0,0 +1 @@ +www.w3.org/2000/09/xmldsig#=src/main/resources/xsds/2000_09_xmldsig#.xsd diff --git a/src/main/resources/xsds/2000_09_xmldsig#.xsd b/src/main/resources/xsds/2000_09_xmldsig#.xsd new file mode 100644 index 0000000..63c09a0 --- /dev/null +++ b/src/main/resources/xsds/2000_09_xmldsig#.xsd @@ -0,0 +1,253 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlCollectionTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlCollectionTest.kt index 4a39e73..3fd68f7 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlCollectionTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlCollectionTest.kt @@ -81,7 +81,7 @@ class XmlCollectionTest : XmlTest() { } @Test - fun `test decode array with self-closing tag`() { + fun `test decode array with self-closing tag in list element`() { val xml = """ @@ -101,4 +101,18 @@ class XmlCollectionTest : XmlTest() { checkDecode(xml, msg) } + + @Test + fun `test decode array with self-closing tag as single element`() { + val xml = """ + + + + """.trimIndent() + val msg = parsedMessage("TestCollection").addFields( + "TestCollection", message(), + ) + + checkDecode(xml, msg) + } } \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlDecodeTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlDecodeTest.kt new file mode 100644 index 0000000..14a4d15 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlDecodeTest.kt @@ -0,0 +1,87 @@ +/* + * Copyright 2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.codec.xml + +import com.exactpro.th2.codec.api.IPipelineCodec +import com.exactpro.th2.codec.api.impl.ReportingContext +import com.exactpro.th2.common.grpc.AnyMessage +import com.exactpro.th2.common.grpc.MessageGroup +import com.exactpro.th2.common.grpc.RawMessage +import com.google.protobuf.ByteString +import com.google.protobuf.util.JsonFormat +import org.junit.jupiter.api.Assertions.assertEquals +import java.io.ByteArrayInputStream +import kotlin.io.path.Path +import kotlin.io.path.readBytes + +class XmlDecodeTest { + + companion object { + private val XML_FILE_PATH = Path("tmp", "test.xml") + private val PROTO_FILE_PATH = Path("tmp", "test_orig.bin") + private const val ITERATIONS = 10_000 + private const val CYCLES = 5 + + @JvmStatic + fun main(args: Array) { + val xml = XML_FILE_PATH.readBytes() + val proto = PROTO_FILE_PATH.readBytes() + val message = AnyMessage.newBuilder().setRawMessage(RawMessage.newBuilder().apply { + metadataBuilder.apply { + protocol = XmlPipelineCodecFactory.PROTOCOL + idBuilder.connectionIdBuilder.sessionAlias = "test_session_alias" + } + body = ByteString.copyFrom(xml) + }) + val messageGroup = MessageGroup.newBuilder().apply { + addMessages(message) + }.build() + + val protoMessageGroup: MessageGroup = MessageGroup.parseFrom(proto) + + XmlPipelineCodecFactory().apply { + init(ByteArrayInputStream(byteArrayOf())) + }.use { factory -> + val codec = factory.create() + + for (cycle in 0 until CYCLES) { + parse(codec, messageGroup) + } + + parse(codec, messageGroup).also { + with(JsonFormat.printer()) { + assertEquals(print(protoMessageGroup), print(it)) + } + } + } + } + + private fun parse( + codec: IPipelineCodec, + messageGroup: MessageGroup + ): MessageGroup { + var millisecond = 0L + lateinit var result: MessageGroup + + for (i in 0..ITERATIONS) { + val start = System.currentTimeMillis() + result = codec.decode(messageGroup, ReportingContext()) + millisecond += System.currentTimeMillis() - start + } + println("Avg ${ITERATIONS.toDouble() / (millisecond) * 1_000}") + return result + } + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecTest.kt index 68b8d23..a48f213 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/XmlPipelineCodecTest.kt @@ -82,9 +82,8 @@ class XmlPipelineCodecTest : XmlTest() { } @Test - fun `test validation of xml declaration`() { - val withoutValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings(expectsDeclaration = false), mapOf()) - val withValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings(expectsDeclaration = true), mapOf()) + fun `test xml declaration`() { + val withoutValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings(), mapOf()) // Message with XML declaration var xml: MessageGroup = createMessageGroup(""" @@ -93,8 +92,7 @@ class XmlPipelineCodecTest : XmlTest() { """.trimIndent()) - Assertions.assertDoesNotThrow { withoutValidationCodec.decode(xml) } - Assertions.assertDoesNotThrow { withValidationCodec.decode(xml) } + Assertions.assertDoesNotThrow { withoutValidationCodec.decode(xml, reportingContext) } // Formatted message with XML declaration xml = createMessageGroup(""" @@ -103,13 +101,12 @@ class XmlPipelineCodecTest : XmlTest() { """.trimIndent()) - Assertions.assertDoesNotThrow { withoutValidationCodec.decode(xml) } - Assertions.assertThrows(IllegalStateException::class.java) { withValidationCodec.decode(xml) } + Assertions.assertDoesNotThrow { withoutValidationCodec.decode(xml, reportingContext) } } @Test - fun `test validation of xml with few root elements`() { - val withoutValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings(expectsDeclaration = false), mapOf()) + fun `test xml with few root elements`() { + val withoutValidationCodec = XmlPipelineCodec(XmlPipelineCodecSettings()) val xml = createMessageGroup(""" @@ -121,7 +118,7 @@ class XmlPipelineCodecTest : XmlTest() { """.trimIndent()) Assertions.assertThrows(IllegalStateException::class.java) { - withoutValidationCodec.decode(xml) + withoutValidationCodec.decode(xml, reportingContext) } } diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/XsdTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/XsdTest.kt index bce6e78..4eb997c 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/xml/XsdTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/XsdTest.kt @@ -62,7 +62,7 @@ class XsdTest : XmlTest() { assertFailsWith ("Error needed due no xsd for xml validation") { - codec.decode(group) + codec.decode(group, reportingContext) } } diff --git a/src/test/kotlin/com/exactpro/th2/codec/xml/utils/XmlTest.kt b/src/test/kotlin/com/exactpro/th2/codec/xml/utils/XmlTest.kt index 6f9eef6..ec0af41 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/xml/utils/XmlTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/xml/utils/XmlTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -16,31 +16,28 @@ package com.exactpro.th2.codec.xml.utils import com.exactpro.th2.codec.api.IPipelineCodec +import com.exactpro.th2.codec.api.impl.ReportingContext import com.exactpro.th2.codec.xml.XmlPipelineCodec -import com.exactpro.th2.codec.xml.XmlPipelineCodecFactory.Companion.decodeInputToDictionary import com.exactpro.th2.codec.xml.XmlPipelineCodecSettings import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.Message import com.exactpro.th2.common.grpc.MessageGroup import com.google.protobuf.TextFormat -import mu.KotlinLogging +import io.github.oshai.kotlinlogging.KotlinLogging import org.apache.commons.io.FileUtils -import org.slf4j.Logger -import java.io.ByteArrayInputStream import java.io.File -import java.nio.file.Path import java.util.Base64 import kotlin.test.assertEquals -abstract class XmlTest(jsonPathToType: String? = null, nameOfXsdResource: String? = null) { - - protected val codec: IPipelineCodec +abstract class XmlTest(pathToType: String? = null) { + protected val reportingContext = ReportingContext() + protected val codec: IPipelineCodec = XmlPipelineCodec(XmlPipelineCodecSettings(pathToType)) protected fun checkEncode(xml: String, message: Message.Builder) { - val group = codec.encode(MessageGroup.newBuilder().addMessages(AnyMessage.newBuilder().setMessage(message)).build()) + val group = codec.encode(MessageGroup.newBuilder().addMessages(AnyMessage.newBuilder().setMessage(message)).build(), reportingContext) assertEquals(1, group.messagesCount) - LOGGER.info("ENCODE_RESULT: ${TextFormat.shortDebugString(group)}") + LOGGER.info { "ENCODE_RESULT: ${TextFormat.shortDebugString(group)}" } assertEquals( "\n$xml", @@ -49,29 +46,19 @@ abstract class XmlTest(jsonPathToType: String? = null, nameOfXsdResource: String } protected fun checkDecode(xml: String, message: Message.Builder) { - val group = codec.decode(createRawMessage(xml)) + val group = codec.decode(createRawMessage(xml), reportingContext) assertEquals(1, group.messagesCount) - LOGGER.info("DECODE_RESULT: ${TextFormat.shortDebugString(group)}") + LOGGER.info { "DECODE_RESULT: ${TextFormat.shortDebugString(group)}" } assertEqualsMessages(message.build(), group.messagesList[0].message, true) } - init { - val xsdMap = nameOfXsdResource?.run { - val zipBase64 = Thread.currentThread().contextClassLoader.getResource(nameOfXsdResource)!! - decodeInputToDictionary(ByteArrayInputStream(encodeFileToBase64Binary(zipBase64.file)), Path.of("tmp").toString()) - } ?: mapOf() - - codec = XmlPipelineCodec(XmlPipelineCodecSettings(jsonPathToType), xsdMap) - } - protected fun encodeFileToBase64Binary(fileName: String): ByteArray { - val file = File(fileName) - return Base64.getEncoder().encode(FileUtils.readFileToByteArray(file)) + return Base64.getEncoder().encode(FileUtils.readFileToByteArray(File(fileName))) } companion object { - private val LOGGER: Logger = KotlinLogging.logger { } + private val LOGGER = KotlinLogging.logger { } } } \ No newline at end of file diff --git a/src/test/resources/log4j2.properties b/src/test/resources/log4j2.properties new file mode 100644 index 0000000..9a3b5b5 --- /dev/null +++ b/src/test/resources/log4j2.properties @@ -0,0 +1,22 @@ +name=Th2Logger + +# Console appender configuration +appender.console.type=Console +appender.console.name=consoleLogger +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n + +# Root logger level +rootLogger.level=INFO + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref=consoleLogger + +logger.th2.name=com.exactpro.th2 +logger.th2.level=INFO + +logger.evolution.name=com.exactpro.evolution +logger.evolution.level=INFO + +logger.cradle.name=com.exactpro.cradle +logger.cradle.level=INFO \ No newline at end of file diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..0c032ab --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,10 @@ + + + + + + + ^pkg:maven/com\.exactpro\.th2/grpc-.*@.*$ + cpe:/a:grpc:grpc + + \ No newline at end of file