From fdccca794942819d4982da8f567de74d1721272c Mon Sep 17 00:00:00 2001
From: Chen Dai <daichen@amazon.com>
Date: Thu, 21 Sep 2023 09:33:21 -0700
Subject: [PATCH] Fix IT and add new IT

Signed-off-by: Chen Dai <daichen@amazon.com>
---
 .../opensearch/flint/spark/FlintSpark.scala   |  2 +-
 .../FlintSparkCoveringIndexITSuite.scala      |  3 ++-
 .../FlintSparkSkippingIndexITSuite.scala      | 27 +++++++++++++++++--
 3 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSpark.scala b/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSpark.scala
index f18357615..5d3ca4257 100644
--- a/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSpark.scala
+++ b/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSpark.scala
@@ -143,7 +143,7 @@ class FlintSpark(val spark: SparkSession) {
             writeFlintIndex(batchDF)
           }
           .start()
-        Some(jobId.toString)
+        Some(jobId.id.toString)
     }
   }
 
diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala
index 20e4dca24..140ecdd77 100644
--- a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala
+++ b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkCoveringIndexITSuite.scala
@@ -56,7 +56,8 @@ class FlintSparkCoveringIndexITSuite extends FlintSparkSuite {
          |        "columnName": "age",
          |        "columnType": "int"
          |     }],
-         |     "source": "default.ci_test"
+         |     "source": "default.ci_test",
+         |     "options": {}
          |   },
          |   "properties": {
          |     "name": {
diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala
index 5d31d8724..7d5598cd5 100644
--- a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala
+++ b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkSkippingIndexITSuite.scala
@@ -78,7 +78,8 @@ class FlintSparkSkippingIndexITSuite extends FlintSparkSuite {
         |        "columnName": "age",
         |        "columnType": "int"
         |     }],
-        |     "source": "default.test"
+        |     "source": "default.test",
+        |     "options": {}
         |   },
         |   "properties": {
         |     "year": {
@@ -102,6 +103,27 @@ class FlintSparkSkippingIndexITSuite extends FlintSparkSuite {
         |   }
         | }
         |""".stripMargin)
+
+    index.get.options shouldBe FlintSparkIndexOptions.empty
+  }
+
+  test("create skipping index with index options successfully") {
+    flint
+      .skippingIndex()
+      .onTable(testTable)
+      .addValueSet("address")
+      .options(FlintSparkIndexOptions(Map(
+        "auto_refresh" -> "true",
+        "refresh_interval" -> "1 Minute",
+        "checkpoint_location" -> "s3a://test/"
+      )))
+      .create()
+
+    val index = flint.describeIndex(testIndex)
+    index shouldBe defined
+    index.get.options.autoRefresh() shouldBe true
+    index.get.options.refreshInterval() shouldBe Some("1 Minute")
+    index.get.options.checkpointLocation() shouldBe Some("s3a://test/")
   }
 
   test("should not have ID column in index data") {
@@ -484,7 +506,8 @@ class FlintSparkSkippingIndexITSuite extends FlintSparkSuite {
          |        "columnName": "struct_col",
          |        "columnType": "struct<subfield1:string,subfield2:int>"
          |     }],
-         |     "source": "$testTable"
+         |     "source": "$testTable",
+         |     "options": {}
          |   },
          |   "properties": {
          |     "boolean_col": {