[Nrm-415] Added unescape-query option.

Dockerfile

@@ -1,4 +1,4 @@
-FROM gradle:7.6-jdk11 AS build
+FROM gradle:8.7-jdk11 AS build
 ARG release_version
 COPY ./ .
 RUN gradle build dockerPrepare \

README.md

@@ -1,4 +1,4 @@
-# th2-codec-oracle-log-miner (0.2.1)
+# th2-codec-oracle-log-miner (0.3.0)
 
 ## Description
 
@@ -66,6 +66,7 @@ save-columns: [ OPERATION, SQL_REDO, ROW_ID, TIMESTAMP, TABLE_NAME ]
 **truncate-update-query-from-where-clause** - if true, codec truncates the tail of UPDATE query starting from the WHERE clause before deep parsing.
 This operation improve performance without negative impact, because codec extracts data from the SET clause only.
 **trim-parsed-content** - if true, Codec trims values parsed from `SQL_REDO` field. Default value is `true`
+**escaped-query** - if `true`, the codec considers `SQL_REDO` field to be escaped XML and performs unescaping it before parsing. The default value is `false`
 **column-prefix** - prefix for parsed columns.
 **save-columns** - set of column names to copy from source message.
 All columns which log miner allow to select are described in the [document](https://docs.oracle.com/en/database/oracle/oracle-database/19/refrn/V-LOGMNR_CONTENTS.html#GUID-B9196942-07BF-4935-B603-FA875064F5C3) 
@@ -79,7 +80,7 @@ metadata:
   name: codec-oracle-log-miner
 spec:
   imageName: ghcr.io/th2-net/th2-codec-oracle-log-miner
-  imageVersion: 0.0.1
+  imageVersion: 0.3.0
   customConfig:
     transportLines:
       rpt:
@@ -92,6 +93,7 @@ spec:
     codecSettings:
       truncate-update-query-from-where-clause: true
       trim-parsed-content: true
+      escaped-query: false
       column-prefix: th2_
       save-columns:
         - OPERATION
@@ -143,6 +145,16 @@ spec:
 
 ## Release notes
 
+### 0.3.0
++ Added `escaped-query` option.
++ Updated: th2 gradle plugin: `0.1.2`
++ Updated-lib:
+  + bom: `4.7.0`
+  + common: `5.14.0-dev`
+  + common-utils: `2.3.0-dev`
+  + kotlin-logging: `5.1.4`
+  + antlr4: `4.13.2`
+
 ### 0.2.1
 + Migrated to th2 gradle plugin: `0.0.8`
 + Updated common: `5.12.0-dev`

build.gradle

@@ -6,8 +6,8 @@ plugins {
     id("java-library")
     id("maven-publish")
 
-    id "com.exactpro.th2.gradle.publish" version "0.0.8"
-    id "com.exactpro.th2.gradle.component" version "0.0.8"
+    id "com.exactpro.th2.gradle.publish" version "0.1.2"
+    id "com.exactpro.th2.gradle.component" version "0.1.2"
 
     id "antlr"
     id "me.champeau.jmh" version "0.7.2"
@@ -39,17 +39,17 @@ repositories {
 }
 
 dependencies {
-    implementation "com.exactpro.th2:common:5.12.0-dev"
-    implementation "com.exactpro.th2:common-utils:2.2.3-dev"
+    implementation "com.exactpro.th2:common:5.14.0-dev"
+    implementation "com.exactpro.th2:common-utils:2.3.0-dev"
     implementation "com.exactpro.th2:codec:5.5.0-dev"
 
     implementation "com.fasterxml.jackson.core:jackson-databind"
+    implementation 'org.apache.commons:commons-text'
 
-    implementation "io.github.microutils:kotlin-logging:3.0.5"
+    implementation 'io.github.oshai:kotlin-logging:5.1.4'
 
-    // https://mvnrepository.com/artifact/org.antlr/antlr4
-    antlr 'org.antlr:antlr4:4.13.1'
-    implementation 'org.antlr:antlr4-runtime:4.13.1'
+    antlr 'org.antlr:antlr4:4.13.2'
+    implementation 'org.antlr:antlr4-runtime:4.13.2'
 
     jmh 'org.openjdk.jmh:jmh-core:1.37'
     jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37'
@@ -59,10 +59,10 @@ dependencies {
     compileOnly "com.google.auto.service:auto-service-annotations:1.1.1"
     kapt "com.google.auto.service:auto-service:1.1.1"
 
-    testImplementation "com.github.doyaaaaaken:kotlin-csv-jvm:1.9.3"
+    testImplementation 'com.jsoizo:kotlin-csv:1.10.0'
     testImplementation "org.jetbrains.kotlin:kotlin-test-junit"
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2'
-    testImplementation 'org.mockito.kotlin:mockito-kotlin:5.3.1'
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0'
+    testImplementation 'org.mockito.kotlin:mockito-kotlin:5.4.0'
     testImplementation 'io.strikt:strikt-core:0.34.1'
 }
 
@@ -74,6 +74,10 @@ configurations {
     }
 }
 
+test {
+    useJUnitPlatform()
+}
+
 application {
     mainClass.set("com.exactpro.th2.codec.MainKt")
 }

gradle.properties

@@ -1,3 +1,3 @@
-release_version=0.2.1
+release_version=0.3.0
 description='th2 codec oracle log miner'
 vcs_url=https://github.com/th2-net/th2-codec-oracle-log-miner

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformer.kt

@@ -31,7 +31,8 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.Message
 import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageGroup
 import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage
 import com.exactpro.th2.common.utils.message.transport.logId
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
+import org.apache.commons.text.StringEscapeUtils.unescapeXml
 import kotlin.contracts.ExperimentalContracts
 import kotlin.contracts.contract
 
@@ -56,7 +57,7 @@ class LogMinerTransformer(private val config: LogMinerConfiguration) : IPipeline
                     }
                     val sqlRedo = requireNotNull(message.body[LOG_MINER_SQL_REDO_COLUMN]?.toString()) {
                         "Message doesn't contain required field '$LOG_MINER_SQL_REDO_COLUMN', id: ${message.id.logId}"
-                    }
+                    }.run { if (config.escapedQuery) unescapeXml(this) else this }
 
                     when (operation) {
                         INSERT.name -> {

src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/antlr/listener/AbstractListener.kt

@@ -20,7 +20,7 @@ import com.exactpro.th2.codec.oracle.logminer.antlr.PlSqlLexer
 import com.exactpro.th2.codec.oracle.logminer.antlr.PlSqlParser
 import com.exactpro.th2.codec.oracle.logminer.antlr.PlSqlParserBaseListener
 import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.MapBuilder
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import org.antlr.v4.runtime.CharStreams
 import org.antlr.v4.runtime.CommonTokenStream
 import org.antlr.v4.runtime.ParserRuleContext

src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/antlr/listener/InsertListener.kt

@@ -23,7 +23,7 @@ import com.exactpro.th2.codec.oracle.logminer.antlr.listener.InsertListener.Stag
 import com.exactpro.th2.codec.oracle.logminer.antlr.listener.InsertListener.Stage.PARSING_NAMES
 import com.exactpro.th2.codec.oracle.logminer.antlr.listener.InsertListener.Stage.PARSING_VALUES
 import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.MapBuilder
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import org.antlr.v4.runtime.ParserRuleContext
 import org.antlr.v4.runtime.tree.ParseTree
 import java.util.LinkedList

src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/antlr/listener/UpdateListener.kt

@@ -18,7 +18,7 @@ package com.exactpro.th2.codec.oracle.logminer.antlr.listener
 
 import com.exactpro.th2.codec.oracle.logminer.antlr.PlSqlParser
 import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.builders.MapBuilder
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import org.antlr.v4.runtime.ParserRuleContext
 import org.antlr.v4.runtime.tree.ParseTree
 

src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/cfg/LogMinerConfiguration.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2023-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,4 +37,8 @@ class LogMinerConfiguration : IPipelineCodecSettings {
     @JsonProperty("trim-parsed-content")
     @JsonPropertyDescription("Codec trims values parsed from SQL_REDO field")
     var trimParsedContent: Boolean = true
+
+    @JsonProperty("escaped-query")
+    @JsonPropertyDescription("Codec unescapes SQL_REDO field by XML standards before parse")
+    var escapedQuery: Boolean = false
 }

src/test/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformerTest.kt

@@ -18,10 +18,10 @@ package com.exactpro.th2.codec.oracle.logminer
 
 import com.exactpro.th2.codec.api.IReportingContext
 import com.exactpro.th2.codec.oracle.logminer.LogMinerTransformer.Companion.truncateFromWhereClause
-import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.INSERT
-import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.UPDATE
 import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.DELETE
+import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.INSERT
 import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.UNSUPPORTED
+import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.UPDATE
 import com.exactpro.th2.codec.oracle.logminer.antlr.listener.InsertListener
 import com.exactpro.th2.codec.oracle.logminer.antlr.listener.UpdateListener
 import com.exactpro.th2.codec.oracle.logminer.cfg.LogMinerConfiguration
@@ -62,11 +62,22 @@ class LogMinerTransformerTest {
         verifyNoMoreInteractions(reportingContext)
     }
 
-    @Test
-    fun decodesDataUsingDefaultHeader() {
-        val config = LogMinerConfiguration().apply { trimParsedContent = false }
+    @ParameterizedTest
+    @CsvSource(
+        "log_miner.csv, true",
+        "log_miner.csv, false",
+        "log_miner_escaped.csv, true",
+    )
+    fun `decodes data using default header`(
+        fileName: String,
+        escapedQuery: String,
+    ) {
+        val config = LogMinerConfiguration().apply {
+            this.trimParsedContent = false
+            this.escapedQuery = escapedQuery.toBoolean()
+        }
         val codec = LogMinerTransformer(config)
-        val sourceMessages: List<ParsedMessage> = loadMessages()
+        val sourceMessages: List<ParsedMessage> = loadMessages(fileName)
         assertEquals(5, sourceMessages.size)
 
         sourceMessages[0].let { source ->
@@ -421,9 +432,9 @@ class LogMinerTransformerTest {
         assertEquals(target, truncateFromWhereClause(source))
     }
 
-    private fun loadMessages(): List<ParsedMessage> {
+    private fun loadMessages(fileName: String): List<ParsedMessage> {
         return LogMinerTransformerTest::class.java.getResourceAsStream(
-            "/com/exactpro/th2/codec/oracle/logminer/log_miner.csv"
+            "/com/exactpro/th2/codec/oracle/logminer/$fileName"
         ).use { inputStream ->
             requireNotNull(inputStream) {
                 "'log_miner.csv' resource doesn't exist"

src/test/resources/com/exactpro/th2/codec/oracle/logminer/log_miner_escaped.csv

@@ -0,0 +1,20 @@
+SCN,START_SCN,COMMIT_SCN,TIMESTAMP,START_TIMESTAMP,COMMIT_TIMESTAMP,XIDUSN,XIDSLT,XIDSQN,XID,PXIDUSN,PXIDSLT,PXIDSQN,PXID,TX_NAME,OPERATION,OPERATION_CODE,ROLLBACK,SEG_OWNER,SEG_NAME,TABLE_NAME,SEG_TYPE,SEG_TYPE_NAME,TABLE_SPACE,ROW_ID,USERNAME,OS_USERNAME,MACHINE_NAME,AUDIT_SESSIONID,SESSION#,SERIAL#,SESSION_INFO,THREAD#,SEQUENCE#,RBASQN,RBABLK,RBABYTE,UBAFIL,UBABLK,UBAREC,UBASQN,ABS_FILE#,REL_FILE#,DATA_BLK#,DATA_OBJ#,DATA_OBJV#,DATA_OBJD#,SQL_REDO,SQL_UNDO,RS_ID,SSN,CSF,INFO,STATUS,REDO_VALUE,UNDO_VALUE,SAFE_RESUME_SCN,CSCN,OBJECT_ID,EDITION_NAME,CLIENT_ID,SRC_CON_NAME,SRC_CON_ID,SRC_CON_UID,SRC_CON_DBID,SRC_CON_GUID,CON_ID
+3627136,,,06-Dec-2023 08:53:57,,,2,17,652,[B@76c52298,2,17,652,[B@bb9ab64,,DDL,5,0,C##TH2ADMIN,EMPLOYEE,EMPLOYEE,2,TABLE,,AAAAAAAAAAAAAAAAAB,C##TH2ADMIN,oracle,525bc69b1bb7,50030,198,33860,login_username=C##TH2ADMIN client_info= OS_username=oracle Machine_name=525bc69b1bb7 OS_terminal=pts/1 OS_process_id=781 OS_program_name=sqlplus@525bc69b1bb7 (TNS V1-V3),1,1,4,38411,300,4,0,0,0,0,0,0,76665,1,0,"CREATE TABLE EMPLOYEE
+   (
+      ID INT ,
+      NAME CHARACTER (100) ,
+      TITLE CHARACTER (100) ,
+      SALARY CHARACTER (50) ,
+      BONUS_STRUCTURE CHARACTER (50) ,
+      TIME_OFF INT ,
+      SICK_TIME INT ,
+      HEALTH_PLAN CHARACTER (100) ,
+      VISION_PLAN CHARACTER (100) ,
+      DENTAL_PLAN CHARACTER (100) ,
+      PLAN INT ,
+      SAVINGS INT
+   );",, 0x000004.0000960b.012c ,0,0,USER DDL (PlSql=0 RecDep=0),0,510660,510661,,,,ORA$BASE,,CDB$ROOT,1,1,0,,1
+3627247,,,06-Dec-2023 08:54:31,,,9,16,680,[B@a52ca2e,9,16,680,[B@1ad8df52,,INSERT,1,0,C##TH2ADMIN,EMPLOYEE,EMPLOYEE,2,TABLE,USERS,AAASt5AAHAAAAFcAAA,C##TH2ADMIN,user,host,50042,326,23200,login_username=C##TH2ADMIN client_info= OS_username=user Machine_name=host OS_terminal=unknown OS_process_id=1234 OS_program_name=JDBC Thin Client,1,1,4,38478,156,4,16791044,19,133,7,7,348,76665,1,76665,"insert into "C##TH2ADMIN"."EMPLOYEE"("ID","NAME","TITLE","SALARY","BONUS_STRUCTURE","TIME_OFF","SICK_TIME","HEALTH_PLAN","VISION_PLAN","DENTAL_PLAN","PLAN","SAVINGS") values ('1','Chris Montgomery ','Manager ','110,000 ','5% Quarterly ','15','5','Blue Cross and Blue Shield ','Aetna Vision ','Delta Dental ','25000','1');","delete from ""C##TH2ADMIN"".""EMPLOYEE"" where ""ID"" = '1' and ""NAME"" = 'Chris Montgomery ' and ""TITLE"" = 'Manager ' and ""SALARY"" = '110,000 ' and ""BONUS_STRUCTURE"" = '5% Quarterly ' and ""TIME_OFF"" = '15' and ""SICK_TIME"" = '5' and ""HEALTH_PLAN"" = 'Blue Cross and Blue Shield ' and ""VISION_PLAN"" = 'Aetna Vision ' and ""DENTAL_PLAN"" = 'Delta Dental ' and ""PLAN"" = '25000' and ""SAVINGS"" = '1' and ROWID = 'AAASt5AAHAAAAFcAAA';", 0x000004.0000964e.009c ,0,0,,0,510718,510719,,,,,,CDB$ROOT,1,1,0,,1
+3627285,,,06-Dec-2023 08:54:31,,,8,21,689,[B@314c8b4a,8,21,689,[B@26d820eb,,UPDATE,3,0,C##TH2ADMIN,EMPLOYEE,EMPLOYEE,2,TABLE,USERS,AAASt5AAHAAAAFcAAA,C##TH2ADMIN,user,host,50042,326,23200,login_username=C##TH2ADMIN client_info= OS_username=user Machine_name=host OS_terminal=unknown OS_process_id=1234 OS_program_name=JDBC Thin Client,1,1,4,38518,16,4,0,0,0,4,7,348,76665,1,76665,update "C##TH2ADMIN"."EMPLOYEE" set "SAVINGS" = '10' where "SAVINGS" = '1' and ROWID = 'AAASt5AAHAAAAFcAAA';,"update ""C##TH2ADMIN"".""EMPLOYEE"" set ""SAVINGS"" = '1' where ""SAVINGS"" = '10' and ROWID = 'AAASt5AAHAAAAFcAAA';", 0x000004.00009676.0010 ,0,0,,0,510808,510809,,,,,,CDB$ROOT,1,1,0,,1
+3627294,,,06-Dec-2023 08:54:31,,,1,29,644,[B@2c43eb8,1,29,644,[B@7d0cc890,,DELETE,2,0,C##TH2ADMIN,EMPLOYEE,EMPLOYEE,2,TABLE,USERS,AAASt5AAHAAAAFcAAA,C##TH2ADMIN,user,host,50042,326,23200,login_username=C##TH2ADMIN client_info= OS_username=user Machine_name=host OS_terminal=unknown OS_process_id=1234 OS_program_name=JDBC Thin Client,1,1,4,38524,16,4,0,0,0,4,7,348,76665,1,76665,"delete from "C##TH2ADMIN"."EMPLOYEE" where "ID" = '1' and "NAME" = 'Chris Montgomery ' and "TITLE" = 'Manager ' and "SALARY" = '110,000 ' and "BONUS_STRUCTURE" = '5% Quarterly ' and "TIME_OFF" = '15' and "SICK_TIME" = '5' and "HEALTH_PLAN" = 'Blue Cross and Blue Shield ' and "VISION_PLAN" = 'Aetna Vision ' and "DENTAL_PLAN" = 'Delta Dental ' and "PLAN" = '25000' and "SAVINGS" = '10' and ROWID = 'AAASt5AAHAAAAFcAAA';","insert into ""C##TH2ADMIN"".""EMPLOYEE""(""ID"",""NAME"",""TITLE"",""SALARY"",""BONUS_STRUCTURE"",""TIME_OFF"",""SICK_TIME"",""HEALTH_PLAN"",""VISION_PLAN"",""DENTAL_PLAN"",""PLAN"",""SAVINGS"") values ('1','Chris Montgomery ','Manager ','110,000 ','5% Quarterly ','15','5','Blue Cross and Blue Shield ','Aetna Vision ','Delta Dental ','25000','10');", 0x000004.0000967c.0010 ,0,0,,0,510826,510827,,,,,,CDB$ROOT,1,1,0,,1
+2583196,2583196,2583196,06-Dec-2023 08:54:32,06-Dec-2023 08:54:33,06-Dec-2023 08:54:34,10,60,676,0A000600A4020000,10,6,676,0A000600A4020000,,UNSUPPORTED,255,0,SYS,SEG$,SEG$,2,TABLE,SYSTEM,AAAAAIAABAAAB6dAAY,UNKNOWN,UNKNOWN,UNKNOWN,0,0,0,UNKNOWN,1,2,2,40538,16,4,16786509,42,199,1,1,7837,14,0,8,Unsupported,Unsupported,0x000002.00009e5a.0010,0,0,Object or Data type Unsupported,2,1682,1683,0,2583197,,,UNKNOWN,CDB$ROOT,1,1,0,,