From d4dd3aa01ddeed46f1fd0eefdc94b2c6dafc7cdc Mon Sep 17 00:00:00 2001 From: Sean Kao Date: Fri, 16 Aug 2024 23:35:07 -0700 Subject: [PATCH] Revert "Lateral eval expressions supported after Spark upgrading (#544) (#561)" This reverts commit c3b6e392efa78fccd83ff66a068289330b3604d1. --- .../flint/spark/ppl/FlintSparkPPLEvalITSuite.scala | 7 ++++++- ppl-spark-integration/README.md | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLEvalITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLEvalITSuite.scala index 19295fbe8..407c2cb3b 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLEvalITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLEvalITSuite.scala @@ -480,7 +480,12 @@ class FlintSparkPPLEvalITSuite comparePlans(logicalPlan, expectedPlan, checkAnalysis = false) } - test("test lateral eval expressions references") { + // +--------------------------------+ + // | Below tests are not supported | + // +--------------------------------+ + // Todo: Upgrading spark version to 3.4.0 and above could fix this test. + // https://issues.apache.org/jira/browse/SPARK-27561 + ignore("test lateral eval expressions references - SPARK-27561 required") { val frame = sql(s""" | source = $testTable | eval col1 = 1, col2 = col1 | fields name, age, col2 | """.stripMargin) diff --git a/ppl-spark-integration/README.md b/ppl-spark-integration/README.md index f73be5bb7..6b8f2ac5c 100644 --- a/ppl-spark-integration/README.md +++ b/ppl-spark-integration/README.md @@ -249,7 +249,7 @@ Assumptions: `a`, `b`, `c` are existing fields in `table` - `source = table | eval n = now() | eval t = unix_timestamp(a) | fields n,t` - `source = table | eval f = a | where f > 1 | sort f | fields a,b,c | head 5` - `source = table | eval f = a * 2 | eval h = f * 2 | fields a,f,h` - - `source = table | eval f = a * 2, h = f * 2 | fields a,f,h` + - `source = table | eval f = a * 2, h = f * 2 | fields a,f,h` (Spark 3.4.0+ required) - `source = table | eval f = a * 2, h = b | stats avg(f) by h` Limitation: Overriding existing field is unsupported, following queries throw exceptions with "Reference 'a' is ambiguous"