Skip to content

Commit

Permalink
update IT tests according to PR's comments
Browse files Browse the repository at this point in the history 
Signed-off-by: YANGDB <[email protected]>
  • Loading branch information
@YANG-DB
YANG-DB committed Dec 10, 2024
1 parent a46494c commit 3fe3adb
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 42 deletions.
12 changes: 6 additions & 6 deletions docs/ppl-lang/functions/ppl-json.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,23 +217,23 @@ A JSON object format.

Example:

os> source=people | eval deleted = json_delete({"a":"valueA", "b":"valueB"}, json_array("a"))
os> source=people | eval deleted = json_delete({"a":"valueA", "b":"valueB"}, array("a"))
fetched rows / total rows = 1/1
+----------------------------------+
| deleted |
+----------------------------------+
| {"a": "valueA" } |
+----------------------------------+

os> source=people | eval eval deleted = json_delete({"a":[{"b":1},{"b":2},{"c":3}]}, json_array("a.b"))
os> source=people | eval deleted = json_delete({"a":[{"b":1},{"b":2},{"c":3}]}, array("a.b"))
fetched rows / total rows = 1/1
+-----------------------------------------------------------+
| deleted |
+-----------------------------------------------------------+
| {"a":[{"c":3}] } |
+-----------------------------------------------------------+

os> source=people | eval `no_action` = json_delete({"a":[{"b":1},{"b":2},{"c":3}]}, json_array("b.b"))
os> source=people | eval `no_action` = json_delete({"a":[{"b":1},{"b":2},{"c":3}]}, array("b.b"))
fetched rows / total rows = 1/1
+-----------------------------------+
| no_action |
Expand Down Expand Up @@ -262,23 +262,23 @@ Append adds the value to the end of the existing array with the following cases:

Example:

os> source=people | eval append = json_append(`{"a":["valueA", "valueB"]}`, json_array('a', 'valueC', 'valueD'))
os> source=people | eval append = json_append(`{"a":["valueA", "valueB"]}`, array('a', 'valueC', 'valueD'))
fetched rows / total rows = 1/1
+-------------------------------------------------+
| append |
+-------------------------------------------------+
| {"a":["valueA", "valueB", "valueC", "valueD"]} |
+-------------------------------------------------+

os> source=people | eval append = json_append(`{"a":[]}`, json_array('a', 'valueC'))
os> source=people | eval append = json_append(`{"a":[]}`, array('a', 'valueC'))
fetched rows / total rows = 1/1
+-----------------------------------------------+
| append |
+-----------------------------------------------+
| {"a":["valueC"]} |
+-----------------------------------------------+

os> source=people | eval append = json_append(`{"root":{ "a":["valueA", "valueB"]}}`, json_array('root', 'valueC')
os> source=people | eval append = json_append(`{"root":{ "a":["valueA", "valueB"]}}`, array('root', 'valueC')
fetched rows / total rows = 1/1
+-----------------------------------------------+
| append |
Expand Down
22 changes: 11 additions & 11 deletions ...c/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLJsonFunctionITSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_delete() function: one key") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_delete('$validJson1',json_array('age')) | head 1 | fields result
| | eval result = json_delete('$validJson1',array('age')) | head 1 | fields result
| """.stripMargin)
assertSameRows(Seq(Row("{\"account_number\":1,\"balance\":39225,\"gender\":\"M\"}")), frame)

Expand All @@ -417,7 +417,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_delete() function: multiple keys") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_delete('$validJson1',json_array('age','gender')) | head 1 | fields result
| | eval result = json_delete('$validJson1',array('age','gender')) | head 1 | fields result
| """.stripMargin)
assertSameRows(Seq(Row("{\"account_number\":1,\"balance\":39225}")), frame)

Expand All @@ -438,7 +438,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_delete() function: nested key") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_delete('$validJson2',json_array('f2.f3')) | head 1 | fields result
| | eval result = json_delete('$validJson2',array('f2.f3')) | head 1 | fields result
| """.stripMargin)
assertSameRows(Seq(Row("{\"f1\":\"abc\",\"f2\":{\"f4\":\"b\"}}")), frame)

Expand All @@ -459,7 +459,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_delete() function: multi depth keys ") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_delete('$validJson5',json_array('teacher', 'student.rank')) | head 1 | fields result
| | eval result = json_delete('$validJson5',array('teacher', 'student.rank')) | head 1 | fields result
| """.stripMargin)
assertSameRows(Seq(Row("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}")), frame)

Expand All @@ -484,7 +484,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_delete() function: key not found") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_delete('$validJson5',json_array('none')) | head 1 | fields result
| | eval result = json_delete('$validJson5',array('none')) | head 1 | fields result
| """.stripMargin)
assertSameRows(
Seq(Row(
Expand All @@ -509,7 +509,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_append() function: add single value") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_append('$validJson7',json_array('teacher', 'Tom')) | head 1 | fields result
| | eval result = json_append('$validJson7',array('teacher', 'Tom')) | head 1 | fields result
| """.stripMargin)
assertSameRows(
Seq(Row(
Expand All @@ -534,7 +534,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_append() function: add single value key not found") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_append('$validJson7',json_array('headmaster', 'Tom')) | head 1 | fields result
| | eval result = json_append('$validJson7',array('headmaster', 'Tom')) | head 1 | fields result
| """.stripMargin)
assertSameRows(
Seq(Row(
Expand All @@ -559,7 +559,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_append() function: add single Object key not found") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_append('$validJson7',json_array('headmaster', '{"name":"Tomy","rank":1}')) | head 1 | fields result
| | eval result = json_append('$validJson7',array('headmaster', '{"name":"Tomy","rank":1}')) | head 1 | fields result
| """.stripMargin)
assertSameRows(
Seq(Row(
Expand Down Expand Up @@ -587,7 +587,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_append() function: add single Object value") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_append('$validJson7',json_array('student', '{"name":"Tomy","rank":5}')) | head 1 | fields result
| | eval result = json_append('$validJson7',array('student', '{"name":"Tomy","rank":5}')) | head 1 | fields result
| """.stripMargin)
assertSameRows(
Seq(Row(
Expand Down Expand Up @@ -615,7 +615,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_append() function: add multi value") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_append('$validJson7',json_array('teacher', 'Tom', 'Walt')) | head 1 | fields result
| | eval result = json_append('$validJson7',array('teacher', 'Tom', 'Walt')) | head 1 | fields result
| """.stripMargin)
assertSameRows(
Seq(Row(
Expand Down Expand Up @@ -643,7 +643,7 @@ class FlintSparkPPLJsonFunctionITSuite
test("test json_append() function: add nested value") {
val frame = sql(s"""
| source = $testTable
| | eval result = json_append('$validJson8',json_array('school.teacher', 'Tom', 'Walt')) | head 1 | fields result
| | eval result = json_append('$validJson8',array('school.teacher', 'Tom', 'Walt')) | head 1 | fields result
| """.stripMargin)
assertSameRows(
Seq(Row(
Expand Down
57 changes: 32 additions & 25 deletions ...scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanJsonFunctionsTranslatorTestSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import org.scalatest.matchers.should.Matchers

import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar}
import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal}
import org.apache.spark.sql.catalyst.expressions.{Alias, EqualTo, Literal}
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project}
import org.apache.spark.sql.types.DataTypes
Expand Down Expand Up @@ -195,35 +195,41 @@ class PPLLogicalPlanJsonFunctionsTranslatorTestSuite
val context = new CatalystPlanContext
val logPlan =
planTransformer.visit(
plan(pplParser, """source=t a = json_delete('{"a":[{"b":1},{"c":2}]}', '["a.b"]')"""),
plan(
pplParser,
"""source=t | eval result = json_delete('{"a":[{"b":1},{"c":2}]}', array('a.b'))"""),
context)

val table = UnresolvedRelation(Seq("t"))
val keysExpression = Literal("""["a.b"]""")
val jsonObjExp = Literal("""{"a":[{"b":1},{"c":2}]}""")
val jsonFunc = visit("json_delete", util.List.of(jsonObjExp, keysExpression))
val filterExpr = EqualTo(UnresolvedAttribute("a"), jsonFunc)
val filterPlan = Filter(filterExpr, table)
val projectList = Seq(UnresolvedStar(None))
val expectedPlan = Project(projectList, filterPlan)
val keysExpression =
UnresolvedFunction("array", Seq(Literal("a.b")), isDistinct = false)
val jsonObjExp =
Literal("""{"a":[{"b":1},{"c":2}]}""")
val jsonFunc =
Alias(visit("json_delete", util.List.of(jsonObjExp, keysExpression)), "result")()
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
val expectedPlan = Project(Seq(UnresolvedStar(None)), eval)
comparePlans(expectedPlan, logPlan, false)
}

test("test json_append()") {
val context = new CatalystPlanContext
val logPlan =
planTransformer.visit(
plan(pplParser, """source=t a = json_append('{"a":[1,2]}', '["a",3]')"""),
plan(
pplParser,
"""source=t | eval result = json_append('{"a":[{"b":1},{"c":2}]}', array('a.b'))"""),
context)

val table = UnresolvedRelation(Seq("t"))
val keysExpression = Literal("""["a",3]""")
val jsonObjExp = Literal("""{"a":[1,2]}""")
val jsonFunc = visit("json_append", util.List.of(jsonObjExp, keysExpression))
val filterExpr = EqualTo(UnresolvedAttribute("a"), jsonFunc)
val filterPlan = Filter(filterExpr, table)
val projectList = Seq(UnresolvedStar(None))
val expectedPlan = Project(projectList, filterPlan)
val keysExpression =
UnresolvedFunction("array", Seq(Literal("a.b")), isDistinct = false)
val jsonObjExp =
Literal("""{"a":[{"b":1},{"c":2}]}""")
val jsonFunc =
Alias(visit("json_append", util.List.of(jsonObjExp, keysExpression)), "result")()
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
val expectedPlan = Project(Seq(UnresolvedStar(None)), eval)
comparePlans(expectedPlan, logPlan, false)
}

Expand All @@ -233,17 +239,18 @@ class PPLLogicalPlanJsonFunctionsTranslatorTestSuite
planTransformer.visit(
plan(
pplParser,
"""source=t a = json_extend('{"a":[{"b":1},{"c":2}]}', '["a",{"c":2}]')"""),
"""source=t | eval result = json_extend('{"a":[{"b":1},{"c":2}]}', array('a.b'))"""),
context)

val table = UnresolvedRelation(Seq("t"))
val keysExpression = Literal("""["a",{"c":2}]""")
val jsonObjExp = Literal("""{"a":[{"b":1},{"c":2}]}""")
val jsonFunc = visit("json_extend", util.List.of(jsonObjExp, keysExpression))
val filterExpr = EqualTo(UnresolvedAttribute("a"), jsonFunc)
val filterPlan = Filter(filterExpr, table)
val projectList = Seq(UnresolvedStar(None))
val expectedPlan = Project(projectList, filterPlan)
val keysExpression =
UnresolvedFunction("array", Seq(Literal("a.b")), isDistinct = false)
val jsonObjExp =
Literal("""{"a":[{"b":1},{"c":2}]}""")
val jsonFunc =
Alias(visit("json_extend", util.List.of(jsonObjExp, keysExpression)), "result")()
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
val expectedPlan = Project(Seq(UnresolvedStar(None)), eval)
comparePlans(expectedPlan, logPlan, false)
}

Expand Down

0 comments on commit 3fe3adb

Please sign in to comment.