[TH2-5141] codec for oracle Log Mainer format (#1)

.github/workflows/ci-unwelcome-words.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: ${{ github.sha }}
+    - name: Checkout tool
+      uses: actions/checkout@v2
+      with:
+        repository: exactpro-th2/ci-github-action
+        ref: master
+        token: ${{ secrets.PAT_CI_ACTION }}
+        path: ci-github-action
+    - name: Run CI action
+      uses: ./ci-github-action
+      with:
+        ref: ${{ github.sha }}

.github/workflows/dev-java-docker-publish.yml

@@ -0,0 +1,19 @@
+name: Dev build and publish Docker distributions to Github Container Registry ghcr.io
+
+on:
+  push:
+    branches-ignore:
+      - master
+      - version-*
+      - dependabot**
+    paths-ignore:
+      - README.md
+
+jobs:
+  build-job:
+    uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main
+    with:
+      build-target: 'Docker'
+      docker-username: ${{ github.actor }}
+    secrets:
+      docker-password: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/dev-release-java-docker-publish.yml

@@ -0,0 +1,18 @@
+name: Build and release Java distributions to sonatype.
+
+on:
+  push:
+    tags:
+      - \d+.\d+.\d+-dev
+
+jobs:
+  build:
+    uses: th2-net/.github/.github/workflows/compound-java.yml@main
+    with:
+      build-target: 'Docker'
+      runsOn: ubuntu-latest
+      gradleVersion: '7'
+      docker-username: ${{ github.actor }}
+      devRelease: true
+    secrets:
+      docker-password: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release-java-docker-publish.yml

@@ -0,0 +1,16 @@
+name: Build and publish Docker distributions to Github Container Registry ghcr.io
+
+on:
+  push:
+    tags:
+      - \d+.\d+.\d+
+
+jobs:
+  build-job:
+    uses: th2-net/.github/.github/workflows/compound-java.yml@main
+    with:
+      build-target: 'Docker'
+      docker-username: ${{ github.actor }}
+    secrets:
+      docker-password: ${{ secrets.GITHUB_TOKEN }}
+

.gitignore

@@ -1,21 +1,8 @@
+# Gradle
 .gradle
-**/build/
-!src/**/build/
+build
+shared
 
-# Ignore Gradle GUI config
-gradle-app.setting
-
-# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)
-!gradle-wrapper.jar
-
-# Avoid ignore Gradle wrappper properties
-!gradle-wrapper.properties
-
-# Cache of project
-.gradletasknamecache
-
-# Eclipse Gradle plugin generated files
-# Eclipse Core
-.project
-# JDT-specific (Eclipse Java Development Tools)
-.classpath
+# Idea
+.idea
+out/

.gitlab-ci.yml

@@ -0,0 +1,7 @@
+variables:
+  APP_NAME: "th2-codec-oracle-log-miner"
+
+include:
+  - project: 'vivarium/th2/pipelines'
+    ref: master
+    file: '/.gitlab-ci-java.yml'

Dockerfile

@@ -0,0 +1,10 @@
+FROM gradle:7.6-jdk11 AS build
+ARG release_version
+COPY ./ .
+RUN gradle build dockerPrepare \
+    -Prelease_version=${release_version}
+
+FROM adoptopenjdk/openjdk11:alpine
+WORKDIR /home
+COPY --from=build /home/gradle/build/docker ./
+ENTRYPOINT ["/home/service/bin/service"]

README.md

@@ -1 +1,140 @@
-# th2-codec-oracle-log-miner
+# th2-codec-oracle-log-miner (0.0.1)
+
+## Description
+
+Designed for transform parsed message at the end of the pipeline
+1. [th2-read-db which pulls Oracle redo log](https://github.com/th2-net/th2-read-db/tree/dev-2/oracle-log-miner.md)
+2. [th2-codec-csv](https://github.com/th2-net/th2-codec-csv)
+
+The codec (transformer) is based on [th2-codec](https://github.com/th2-net/th2-codec). You can find additional information [here](https://github.com/th2-net/th2-codec/blob/master/README.md)
+
+## Decoding (transformation)
+
+The codec parses a SQL query in the `SQL_REDO` field to extract column and value pairs and appends them into an output message.
+The `OPERATION`, `SQL_REDO`, `ROW_ID`, `TIMESTAMP`, `TABLE_NAME` fields are required in source parsed message for transformation otherwise `th2-codec-error` message is produced 
+
+### Parser workflow:
+1. detects type of query by the `OPERATION` field. 
+Currently, the `INSERT` / `UPDATE` / `DELETE` values are supported.
+2. parses a value of the `SQL_REDO` field as SQL query.
+3. creates an empty parsed message and copies the whole metadata from the source message.
+4. appends parsed columns and values to the new parsed message. 
+Each field will have a prefix specified in the `columnPrefix` option.
+5. copies fields specified the `saveColumns` option from source to new parsed message.
+
+### Transform example:
+
+source parsed message body
+```json
+{
+  "SCN": 3627285,
+  "TIMESTAMP": "06-Dec-2023 08:54:31",
+  "OPERATION": "UPDATE",
+  "TABLE_NAME": "EMPLOYEE",
+  "ROW_ID": "AAASt5AAHAAAAFcAAA",
+  "SQL_REDO": "update \"<user>\".\"EMPLOYEE\" set \"SAVINGS\" = '10' where \"SAVINGS\" = '1' and ROWID = 'AAASt5AAHAAAAFcAAA';"
+}
+```
+
+outgoing parsed message body after transform with default configuration
+```json
+{
+  "TIMESTAMP": "06-Dec-2023 08:54:31",
+  "OPERATION": "UPDATE",
+  "TABLE_NAME": "EMPLOYEE",
+  "ROW_ID": "AAASt5AAHAAAAFcAAA",
+  "SQL_REDO": "update \"<user>\".\"EMPLOYEE\" set \"SAVINGS\" = '10' where \"SAVINGS\" = '1' and ROWID = 'AAASt5AAHAAAAFcAAA';",
+  "th2_SAVINGS": "10"
+}
+```
+
+## Encode
+
+This operation isn't support.
+
+## Settings
+
+Oracle log miner codec (transformer) has the following parameters:
+```yaml
+columnPrefix: 'th2_'
+saveColumns: [ OPERATION, SQL_REDO, ROW_ID, TIMESTAMP, TABLE_NAME ]
+```
+
+**columnPrefix** - prefix for parsed columns.
+**saveColumns** - set of column names to copy from source message.
+All columns which log miner allow to select are described in the [document](https://docs.oracle.com/en/database/oracle/oracle-database/19/refrn/V-LOGMNR_CONTENTS.html#GUID-B9196942-07BF-4935-B603-FA875064F5C3) 
+
+## Full configuration example
+
+```yaml
+apiVersion: th2.exactpro.com/v2
+kind: Th2Box
+metadata:
+  name: codec-oracle-log-miner
+spec:
+  imageName: ghcr.io/th2-net/th2-codec-oracle-log-miner
+  imageVersion: 0.0.1
+  customConfig:
+    transportLines:
+      rpt:
+        type: TH2_TRANSPORT
+        useParentEventId: false    
+    # required true to decode (transform) parsed message 
+    disableMessageTypeCheck: true
+    disableProtocolCheck: true
+
+    codecSettings:
+      column-prefix: th2_
+      save-columns:
+        - OPERATION
+        - SQL_REDO
+        - TIMESTAMP
+        - ROW_ID
+        - TABLE_NAME
+  pins:
+    grpc:
+      server:
+        - name: server
+          serviceClasses:
+            - com.exactpro.th2.codec.grpc.CodecService
+    mq:
+      subscribers:
+        - name: lwdp_in_codec_encode
+          attributes: [ transport-group, subscribe, lwdp_encoder_in ]
+        - name: lwdp_in_oracle_log_miner
+          attributes:
+            - transport-group
+            - subscribe
+            - lwdp_decoder_in
+
+        - name: rpt_in_codec_encode
+          attributes: [ transport-group, subscribe, rpt_encoder_in ]
+        - name: rpt_in_oracle_log_miner
+          attributes:
+            - transport-group
+            - subscribe
+            - rpt_decoder_in
+
+      publishers:
+        - name: lwdp_out_codec_encode
+          attributes: [ transport-group, publish, lwdp_encoder_out ]
+        - name: lwdp_out_codec_decode
+          attributes:
+            - transport-group
+            - publish
+            - lwdp_decoder_out
+
+        - name: rpt_out_codec_encode
+          attributes: [ transport-group, publish, rpt_encoder_out ]
+        - name: rpt_out_codec_decode
+          attributes:
+            - transport-group
+            - publish
+            - rpt_decoder_out
+```
+
+## Release notes
+
+### 0.0.1
++ Parse `INSERT` / `UPDATE` / `DELETE` SQL queries from the `SQL_REDO` field.
++ Copy specified fields from source to output parsed message.

build.gradle

@@ -0,0 +1,32 @@
+buildscript {
+    repositories {
+        gradlePluginPortal()
+        maven {
+            url = "https://s01.oss.sonatype.org/content/repositories/snapshots/"
+        }
+    }
+    dependencies {
+        classpath "com.exactpro.th2:th2-gradle-plugin:0.0.1-dev-5915772757-13a28ae-SNAPSHOT"
+    }
+}
+
+apply plugin: "com.exactpro.th2.common-conventions"
+apply plugin: "com.exactpro.th2.docker-conventions"
+
+apply plugin: 'kotlin-kapt'
+
+dependencies {
+    api platform("com.exactpro.th2:bom:4.5.0")
+    implementation "com.exactpro.th2:common:5.7.2-dev"
+    implementation "com.exactpro.th2:common-utils:2.2.2-dev"
+    implementation "com.exactpro.th2:codec:5.4.1-dev"
+
+    implementation "com.fasterxml.jackson.core:jackson-databind"
+    implementation 'com.github.jsqlparser:jsqlparser:4.7'
+
+    compileOnly "com.google.auto.service:auto-service-annotations:1.1.1"
+    annotationProcessor "com.google.auto.service:auto-service:1.1.1"
+    kapt "com.google.auto.service:auto-service:1.1.1"
+
+    testImplementation "com.github.doyaaaaaken:kotlin-csv-jvm:1.9.2"
+}

gradle.properties

@@ -0,0 +1,2 @@
+release_version=0.0.1
+app_main_class=com.exactpro.th2.codec.MainKt

gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-all.zip
+networkTimeout=10000
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists