Add support for serializing MapType (#929) (#939)

(cherry picked from commit 8efe1ec) Signed-off-by: Chase Engelbrecht <[email protected]> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Nov 21, 2024 · 5585f1a · 5585f1a
1 parent 12b8091
commit 5585f1a
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 0 deletions.
diff --git a/...-spark-integration/src/main/scala/org/apache/spark/sql/flint/datatype/FlintDataType.scala b/...-spark-integration/src/main/scala/org/apache/spark/sql/flint/datatype/FlintDataType.scala
@@ -153,6 +153,9 @@ object FlintDataType {
       // objects
       case st: StructType => serializeJValue(st)
 
+      // Serialize maps as empty objects and let the map entries automap
+      case mt: MapType => serializeJValue(new StructType())
+
       // array
       case ArrayType(elementType, _) => serializeField(elementType, Metadata.empty)
 

diff --git a/...k-integration/src/test/scala/org/apache/spark/sql/flint/datatype/FlintDataTypeSuite.scala b/...k-integration/src/test/scala/org/apache/spark/sql/flint/datatype/FlintDataTypeSuite.scala
@@ -128,6 +128,21 @@ class FlintDataTypeSuite extends FlintSuite with Matchers {
                                                                     |}""".stripMargin)
   }
 
+  test("spark map type serialize") {
+    val sparkStructType = StructType(
+      StructField("mapField", MapType(StringType, StringType), true) ::
+        Nil)
+
+    FlintDataType.serialize(sparkStructType) shouldBe compactJson("""{
+                                                                    |  "properties": {
+                                                                    |    "mapField": {
+                                                                    |      "properties": {
+                                                                    |      }
+                                                                    |    }
+                                                                    |  }
+                                                                    |}""".stripMargin)
+  }
+
   test("spark varchar and char type serialize") {
     val flintDataType = """{
                           |  "properties": {