From 2864b285f0d22cc778dceef0735755df2f1c1502 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Fri, 17 May 2024 13:53:51 -0700 Subject: [PATCH] Update user manual Signed-off-by: Chen Dai --- docs/index.md | 3 +++ .../flint/spark/FlintSparkIndexMonitorITSuite.scala | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 7900cd359..46ebd87eb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -519,6 +519,9 @@ In the index mapping, the `_meta` and `properties`field stores meta and schema i - `spark.flint.index.hybridscan.enabled`: default is false. - `spark.flint.index.checkpoint.mandatory`: default is true. - `spark.datasource.flint.socket_timeout_millis`: default value is 60000. +- `spark.flint.monitor.initialDelaySeconds`: Initial delay in seconds before starting the monitoring task. Default value is 15. +- `spark.flint.monitor.intervalSeconds`: Interval in seconds for scheduling the monitoring task. Default value is 60. +- `spark.flint.monitor.maxErrorCount`: Maximum number of consecutive errors allowed before stopping the monitoring task. Default value is 5. #### Data Type Mapping diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexMonitorITSuite.scala b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexMonitorITSuite.scala index 39b49cf77..01dc63e00 100644 --- a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexMonitorITSuite.scala +++ b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkIndexMonitorITSuite.scala @@ -140,7 +140,11 @@ class FlintSparkIndexMonitorITSuite extends OpenSearchTransactionSuite with Matc waitForMonitorTaskRun() // Both monitor task and streaming job should stop after 10 times - 10 times { (_, _) => {} } + 10 times { (_, _) => + { + // assert nothing. just wait enough times of task execution + } + } task.isCancelled shouldBe true spark.streams.active.exists(_.name == testFlintIndex) shouldBe false