Make sure spark option keys are case-insensitive when passed to the o…

…ption parser.
AbsaOSS · Apr 26, 2024 · 4b141ef · 4b141ef
1 parent 874b772
commit 4b141ef
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 2 deletions.
diff --git a/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/schema/CobolSchema.scala b/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/schema/CobolSchema.scala
@@ -336,7 +336,8 @@ object CobolSchema {
   }
 
   def fromSparkOptions(copyBookContents: Seq[String], sparkReaderOptions: Map[String, String]): CobolSchema = {
-    val cobolParameters = CobolParametersParser.parse(new Parameters(sparkReaderOptions))
+    val lowercaseOptions = sparkReaderOptions.map { case (k, v) => (k.toLowerCase, v) }
+    val cobolParameters = CobolParametersParser.parse(new Parameters(lowercaseOptions))
     val readerParameters = getReaderProperties(cobolParameters, None)
 
     CobolSchema.fromBaseReader(CobolReaderSchema.fromReaderParameters(copyBookContents, readerParameters))

diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala
@@ -549,6 +549,24 @@ class CobolSchemaSpec extends AnyWordSpec with SimpleComparisonBase {
       assert(sparkSchema.fields(8).dataType.isInstanceOf[StructType])
     }
 
+    "work in the case insensitive way" in {
+      val copybook: String =
+        """       01  RECORD.
+          |         05  STR1                  PIC X(10).
+          |""".stripMargin
+
+      val cobolSchema = CobolSchema.fromSparkOptions(Seq(copybook),
+        Map(
+          "pedantic" -> "true",
+          "generate_RECORD_id" -> "true"
+        )
+      )
+
+      val sparkSchema = cobolSchema.getSparkSchema
+
+      assert(sparkSchema.fields.length == 4)
+    }
+
     "fail on redundant options when pedantic mode is turned on" in {
       val copybook: String =
         """       01  RECORD.

diff --git a/.../test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test22RecordLengthGenId.scala b/.../test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test22RecordLengthGenId.scala
@@ -49,7 +49,7 @@ class Test22RecordLengthGenId extends AnyWordSpec with SparkTestBase with Binary
           .format("cobol")
           .option("copybook_contents", copybook)
           .option("record_format", "F")
-          .option("record_length", "4")
+          .option("record_LENGTH", "4")
           .option("pedantic", "true")
           .load(tmpFileName)