From 351614d934f343841eb85e12bc5aa19fac674b85 Mon Sep 17 00:00:00 2001 From: Kaituo Li Date: Tue, 14 Nov 2023 12:33:02 -0800 Subject: [PATCH] Refactor 'excludeJobIds' Serialization for Spark Job Compatibility This commit changes the serialization format of the 'excludeJobIds' field in the session document. Previously serialized as an array (e.g., ["id1", "id2"]), this field is now formatted as a comma-separated string ("id1,id2"). This adjustment is made to ensure compatibility with plugins that currently do not support array parsing. This simplification aids in seamless integration and easier data handling in downstream processes. Testing done: * modified unit test Signed-off-by: Kaituo Li --- .../scala/org/opensearch/flint/app/FlintInstance.scala | 7 ++++++- .../scala/org/opensearch/flint/app/FlintInstanceTest.scala | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/flint-spark-integration/src/main/scala/org/opensearch/flint/app/FlintInstance.scala b/flint-spark-integration/src/main/scala/org/opensearch/flint/app/FlintInstance.scala index aa417e61a..f9e8dd693 100644 --- a/flint-spark-integration/src/main/scala/org/opensearch/flint/app/FlintInstance.scala +++ b/flint-spark-integration/src/main/scala/org/opensearch/flint/app/FlintInstance.scala @@ -120,7 +120,12 @@ object FlintInstance { "state" -> job.state, // update last update time "lastUpdateTime" -> currentTime, - "excludeJobIds" -> job.excludedJobIds, + // Convert a Seq[String] into a comma-separated string, such as "id1,id2". + // This approach is chosen over serializing to an array format (e.g., ["id1", "id2"]) + // because it simplifies client-side processing. With a comma-separated string, + // clients can easily ignore this field if it's not in use, avoiding the need + // for array parsing logic. This makes the serialized data more straightforward to handle. + "excludeJobIds" -> job.excludedJobIds.mkString(","), "jobStartTime" -> job.jobStartTime)) } } diff --git a/flint-spark-integration/src/test/scala/org/opensearch/flint/app/FlintInstanceTest.scala b/flint-spark-integration/src/test/scala/org/opensearch/flint/app/FlintInstanceTest.scala index caf5a84d4..12c2ae5bc 100644 --- a/flint-spark-integration/src/test/scala/org/opensearch/flint/app/FlintInstanceTest.scala +++ b/flint-spark-integration/src/test/scala/org/opensearch/flint/app/FlintInstanceTest.scala @@ -46,7 +46,9 @@ class FlintInstanceTest extends SparkFunSuite with Matchers { json should include(""""sessionId":"session-789"""") json should include(""""state":"RUNNING"""") json should include(s""""lastUpdateTime":$currentTime""") - json should include(""""excludeJobIds":["job-101","job-202"]""") + json should include( + """"excludeJobIds":"job-101,job-202"""" + ) // Check for comma-separated string json should include(""""jobStartTime":1620000001000""") json should include(""""error":""""") }