From 97a3c4bdb65fae248cdf09e8129c547e0841803a Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Mon, 16 Oct 2023 09:04:35 +0200 Subject: [PATCH 1/5] Setting version to 2.6.9 --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index affd15f2..4a07ad04 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "2.6.9-SNAPSHOT" +ThisBuild / version := "2.6.9" From 412a9b68164e0191f172d9a78c0046dcd15d8f4f Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Mon, 16 Oct 2023 09:08:04 +0200 Subject: [PATCH 2/5] Setting version to 2.6.10-SNAPSHOT --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index 4a07ad04..000cecb5 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "2.6.9" +ThisBuild / version := "2.6.10-SNAPSHOT" From 033dd44ecf8c22dcdde5f10e0f9f5975cb714696 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Mon, 16 Oct 2023 09:41:13 +0200 Subject: [PATCH 3/5] Update README. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index a031fcda..a3d89de5 100644 --- a/README.md +++ b/README.md @@ -1643,6 +1643,10 @@ at org.apache.hadoop.io.nativeio.NativeIO$POSIX.getStat(NativeIO.java:608) A: Update hadoop dll to version 3.2.2 or newer. ## Changelog +- #### 2.6.8 released 16 October 2023. + - [#634](https://github.com/AbsaOSS/cobrix/issues/634) Retain metadata when flattening the schema in SparkUtils. + - [#644](https://github.com/AbsaOSS/cobrix/issues/644) Add support for Spark 3.5.0. + - #### 2.6.8 released 1 June 2023. - [#624](https://github.com/AbsaOSS/cobrix/issues/624) Add support for binary fields that have `PIC X` and `USAGE COMP`. From 246c187b9653368937ae2708f061ef7d54b50023 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Mon, 16 Oct 2023 10:10:05 +0200 Subject: [PATCH 4/5] Update Cobrix version in POM and examples. --- README.md | 24 +++++++++++----------- cobol-converters/pom.xml | 2 +- cobol-parser/pom.xml | 2 +- examples/examples-collection/pom.xml | 2 +- examples/spark-cobol-app/build.sbt | 2 +- examples/spark-cobol-app/pom.xml | 2 +- examples/spark-cobol-s3-standalone/pom.xml | 2 +- examples/spark-cobol-s3/pom.xml | 2 +- pom.xml | 2 +- spark-cobol/pom.xml | 2 +- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index a3d89de5..f89b71e6 100644 --- a/README.md +++ b/README.md @@ -74,13 +74,13 @@ You can link against this library in your program at the following coordinates: -
groupId: za.co.absa.cobrix
artifactId: spark-cobol_2.11
version: 2.6.8
+
groupId: za.co.absa.cobrix
artifactId: spark-cobol_2.11
version: 2.6.9
-
groupId: za.co.absa.cobrix
artifactId: spark-cobol_2.12
version: 2.6.8
+
groupId: za.co.absa.cobrix
artifactId: spark-cobol_2.12
version: 2.6.9
-
groupId: za.co.absa.cobrix
artifactId: spark-cobol_2.13
version: 2.6.8
+
groupId: za.co.absa.cobrix
artifactId: spark-cobol_2.13
version: 2.6.9
@@ -91,17 +91,17 @@ This package can be added to Spark using the `--packages` command line option. F ### Spark compiled with Scala 2.11 ``` -$SPARK_HOME/bin/spark-shell --packages za.co.absa.cobrix:spark-cobol_2.11:2.6.8 +$SPARK_HOME/bin/spark-shell --packages za.co.absa.cobrix:spark-cobol_2.11:2.6.9 ``` ### Spark compiled with Scala 2.12 ``` -$SPARK_HOME/bin/spark-shell --packages za.co.absa.cobrix:spark-cobol_2.12:2.6.8 +$SPARK_HOME/bin/spark-shell --packages za.co.absa.cobrix:spark-cobol_2.12:2.6.9 ``` ### Spark compiled with Scala 2.13 ``` -$SPARK_HOME/bin/spark-shell --packages za.co.absa.cobrix:spark-cobol_2.13:2.6.8 +$SPARK_HOME/bin/spark-shell --packages za.co.absa.cobrix:spark-cobol_2.13:2.6.9 ``` ## Usage @@ -238,17 +238,17 @@ to decode various binary formats. The jars that you need to get are: -* spark-cobol_2.12-2.6.8.jar -* cobol-parser_2.12-2.6.8.jar +* spark-cobol_2.12-2.6.9.jar +* cobol-parser_2.12-2.6.9.jar * scodec-core_2.12-1.10.3.jar * scodec-bits_2.12-1.1.4.jar * antlr4-runtime-4.8.jar After that you can specify these jars in `spark-shell` command line. Here is an example: ``` -$ spark-shell --packages za.co.absa.cobrix:spark-cobol_2.12:2.6.8 +$ spark-shell --packages za.co.absa.cobrix:spark-cobol_2.12:2.6.9 or -$ spark-shell --master yarn --deploy-mode client --driver-cores 4 --driver-memory 4G --jars spark-cobol_2.12-2.6.8.jar,cobol-parser_2.12-2.6.8.jar,scodec-core_2.12-1.10.3.jar,scodec-bits_2.12-1.1.4.jar,antlr4-runtime-4.8.jar +$ spark-shell --master yarn --deploy-mode client --driver-cores 4 --driver-memory 4G --jars spark-cobol_2.12-2.6.9.jar,cobol-parser_2.12-2.6.9.jar,scodec-core_2.12-1.10.3.jar,scodec-bits_2.12-1.1.4.jar,antlr4-runtime-4.8.jar Setting default log level to "WARN". To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). @@ -319,7 +319,7 @@ The fat jar will have '-bundle' suffix. You can also download pre-built bundles Then, run `spark-shell` or `spark-submit` adding the fat jar as the option. ```sh -$ spark-shell --jars spark-cobol_2.12_3.3.2-2.6.9-SNAPSHOT-bundle.jar +$ spark-shell --jars spark-cobol_2.12_3.3.2-2.6.10-SNAPSHOT-bundle.jar ``` > A note for building and running tests on Windows @@ -1643,7 +1643,7 @@ at org.apache.hadoop.io.nativeio.NativeIO$POSIX.getStat(NativeIO.java:608) A: Update hadoop dll to version 3.2.2 or newer. ## Changelog -- #### 2.6.8 released 16 October 2023. +- #### 2.6.9 released 16 October 2023. - [#634](https://github.com/AbsaOSS/cobrix/issues/634) Retain metadata when flattening the schema in SparkUtils. - [#644](https://github.com/AbsaOSS/cobrix/issues/644) Add support for Spark 3.5.0. diff --git a/cobol-converters/pom.xml b/cobol-converters/pom.xml index bf47dd7b..fdda48d9 100644 --- a/cobol-converters/pom.xml +++ b/cobol-converters/pom.xml @@ -22,7 +22,7 @@ za.co.absa.cobrix cobrix_2.12 - 2.6.9-SNAPSHOT + 2.6.10-SNAPSHOT ../pom.xml diff --git a/cobol-parser/pom.xml b/cobol-parser/pom.xml index 195e3320..4780c37f 100644 --- a/cobol-parser/pom.xml +++ b/cobol-parser/pom.xml @@ -22,7 +22,7 @@ za.co.absa.cobrix cobrix_2.12 - 2.6.9-SNAPSHOT + 2.6.10-SNAPSHOT ../pom.xml diff --git a/examples/examples-collection/pom.xml b/examples/examples-collection/pom.xml index ca75e45c..d8b6e59a 100644 --- a/examples/examples-collection/pom.xml +++ b/examples/examples-collection/pom.xml @@ -31,7 +31,7 @@ 2.11 2.4.8 2.4.16 - 2.6.8 + 2.6.9 diff --git a/examples/spark-cobol-app/build.sbt b/examples/spark-cobol-app/build.sbt index 72c25a19..a5f88d37 100644 --- a/examples/spark-cobol-app/build.sbt +++ b/examples/spark-cobol-app/build.sbt @@ -20,7 +20,7 @@ ThisBuild / version := "0.1.0-SNAPSHOT" ThisBuild / scalaVersion := "2.12.17" val sparkVersion = "3.3.2" -val sparkCobolVersion = "2.6.8" +val sparkCobolVersion = "2.6.9" val scalatestVersion = "3.2.14" ThisBuild / libraryDependencies ++= Seq( diff --git a/examples/spark-cobol-app/pom.xml b/examples/spark-cobol-app/pom.xml index 86cf0a45..9c9a5b14 100755 --- a/examples/spark-cobol-app/pom.xml +++ b/examples/spark-cobol-app/pom.xml @@ -31,7 +31,7 @@ 2.12 3.2.14 3.3.2 - 2.6.8 + 2.6.9 diff --git a/examples/spark-cobol-s3-standalone/pom.xml b/examples/spark-cobol-s3-standalone/pom.xml index ab2bc67d..f38aa9f2 100644 --- a/examples/spark-cobol-s3-standalone/pom.xml +++ b/examples/spark-cobol-s3-standalone/pom.xml @@ -32,7 +32,7 @@ 2.11 3.2.3 2.4.8 - 2.6.8 + 2.6.9 3.2.4 diff --git a/examples/spark-cobol-s3/pom.xml b/examples/spark-cobol-s3/pom.xml index 74b91e03..eeb049f1 100644 --- a/examples/spark-cobol-s3/pom.xml +++ b/examples/spark-cobol-s3/pom.xml @@ -32,7 +32,7 @@ 2.11 3.2.14 2.4.8 - 2.6.8 + 2.6.9 diff --git a/pom.xml b/pom.xml index 683e0abc..dfb9f8e4 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ za.co.absa.cobrix cobrix_2.12 - 2.6.9-SNAPSHOT + 2.6.10-SNAPSHOT pom diff --git a/spark-cobol/pom.xml b/spark-cobol/pom.xml index b3bcaa3e..bdcea87c 100644 --- a/spark-cobol/pom.xml +++ b/spark-cobol/pom.xml @@ -22,7 +22,7 @@ za.co.absa.cobrix cobrix_2.12 - 2.6.9-SNAPSHOT + 2.6.10-SNAPSHOT ../pom.xml From 12347734ed934f58311a19ec275075cc16fd5255 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Mon, 16 Oct 2023 10:13:50 +0200 Subject: [PATCH 5/5] Update Spark version in jacoco CI step. --- .github/workflows/jacoco_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 9a0414d7..ef0759bb 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -37,9 +37,9 @@ jobs: # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and # vendor distributions. include: - - scala: 2.12.17 + - scala: 2.12.18 scalaShort: "2.12" - spark: 3.3.2 + spark: 3.3.3 overall: 0.0 changed: 80.0