diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLPatternsITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLPatternsITSuite.scala index 38e4f5544..422ef66c3 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLPatternsITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLPatternsITSuite.scala @@ -5,11 +5,11 @@ package org.opensearch.flint.spark.ppl +import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar} import org.apache.spark.sql.catalyst.expressions.{Alias, Descending, GreaterThan, Literal, NullsLast, RegExpExtract, RegExpReplace, SortOrder} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.streaming.StreamTest -import org.apache.spark.sql.{QueryTest, Row} class FlintSparkPPLPatternsITSuite extends QueryTest @@ -65,11 +65,7 @@ class FlintSparkPPLPatternsITSuite val emailAttribute = UnresolvedAttribute("email") val patterns_field = UnresolvedAttribute("patterns_field") val hostExpression = Alias( - RegExpReplace( - emailAttribute, - Literal( - "[a-zA-Z0-9]"), - Literal("")), + RegExpReplace(emailAttribute, Literal("[a-zA-Z0-9]"), Literal("")), "patterns_field")() val expectedPlan = Project( Seq(emailAttribute, patterns_field), @@ -102,11 +98,7 @@ class FlintSparkPPLPatternsITSuite val patterns_fieldAttribute = UnresolvedAttribute("patterns_field") val ageAttribute = UnresolvedAttribute("age") val patternExpression = Alias( - RegExpReplace( - emailAttribute, - Literal( - "[a-zA-Z0-9]"), - Literal("")), + RegExpReplace(emailAttribute, Literal("[a-zA-Z0-9]"), Literal("")), "patterns_field")() // Define the corrected expected plan @@ -124,7 +116,8 @@ class FlintSparkPPLPatternsITSuite } test("test patterns email expressions and top count_host ") { - val frame = sql("source=spark_catalog.default.flint_ppl_test | patterns new_field='dot_com' pattern='(.com|.net|.org)' email | stats count() by dot_com ") + val frame = sql( + "source=spark_catalog.default.flint_ppl_test | patterns new_field='dot_com' pattern='(.com|.net|.org)' email | stats count() by dot_com ") // Retrieve the results val results: Array[Row] = frame.collect() @@ -144,17 +137,13 @@ class FlintSparkPPLPatternsITSuite // Sort both the results and the expected results implicit val rowOrdering: Ordering[Row] = Ordering.by(r => (r.getLong(0), r.getString(1))) assert(results.sorted.sameElements(expectedResults.sorted)) - + // Retrieve the logical plan val logicalPlan: LogicalPlan = frame.queryExecution.logical val messageAttribute = UnresolvedAttribute("email") val noNumbersAttribute = UnresolvedAttribute("dot_com") val hostExpression = Alias( - RegExpReplace( - messageAttribute, - Literal( - "(.com|.net|.org)"), - Literal("")), + RegExpReplace(messageAttribute, Literal("(.com|.net|.org)"), Literal("")), "dot_com")() // Define the corrected expected plan diff --git a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanPatternsTranslatorTestSuite.scala b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanPatternsTranslatorTestSuite.scala index 1a0223a9c..a26d365d2 100644 --- a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanPatternsTranslatorTestSuite.scala +++ b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanPatternsTranslatorTestSuite.scala @@ -5,15 +5,15 @@ package org.opensearch.flint.spark.ppl +import org.opensearch.flint.spark.ppl.PlaneUtils.plan +import org.opensearch.sql.ppl.{CatalystPlanContext, CatalystQueryPlanVisitor} +import org.scalatest.matchers.should.Matchers + import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar} import org.apache.spark.sql.catalyst.expressions.{Alias, Descending, GreaterThan, Literal, NullsLast, RegExpExtract, RegExpReplace, SortOrder} import org.apache.spark.sql.catalyst.plans.PlanTest import org.apache.spark.sql.catalyst.plans.logical._ -import org.opensearch.flint.spark.ppl.PlaneUtils.plan -import org.opensearch.sql.ppl.{CatalystPlanContext, CatalystQueryPlanVisitor} -import org.scalatest.matchers.should.Matchers - class PPLLogicalPlanPatternsTranslatorTestSuite extends SparkFunSuite @@ -37,11 +37,7 @@ class PPLLogicalPlanPatternsTranslatorTestSuite val emailAttribute = UnresolvedAttribute("email") val patterns_field = UnresolvedAttribute("patterns_field") val hostExpression = Alias( - RegExpReplace( - emailAttribute, - Literal( - "[a-zA-Z0-9]"), - Literal("")), + RegExpReplace(emailAttribute, Literal("[a-zA-Z0-9]"), Literal("")), "patterns_field")() val expectedPlan = Project( Seq(emailAttribute, patterns_field), @@ -51,7 +47,8 @@ class PPLLogicalPlanPatternsTranslatorTestSuite assert(compareByString(expectedPlan) === compareByString(logPlan)) } - test("test patterns extract punctuations from a raw log field using user defined patterns and a new field") { + test( + "test patterns extract punctuations from a raw log field using user defined patterns and a new field") { val context = new CatalystPlanContext val logPlan = planTransformer.visit( @@ -63,13 +60,8 @@ class PPLLogicalPlanPatternsTranslatorTestSuite val emailAttribute = UnresolvedAttribute("message") val patterns_field = UnresolvedAttribute("no_numbers") - val hostExpression = Alias( - RegExpReplace( - emailAttribute, - Literal( - "[0-9]"), - Literal("")), - "no_numbers")() + val hostExpression = + Alias(RegExpReplace(emailAttribute, Literal("[0-9]"), Literal("")), "no_numbers")() val expectedPlan = Project( Seq(emailAttribute, patterns_field), Project( @@ -94,11 +86,7 @@ class PPLLogicalPlanPatternsTranslatorTestSuite val patterns_fieldAttribute = UnresolvedAttribute("patterns_field") val ageAttribute = UnresolvedAttribute("age") val hostExpression = Alias( - RegExpReplace( - emailAttribute, - Literal( - "[a-zA-Z0-9]"), - Literal("")), + RegExpReplace(emailAttribute, Literal("[a-zA-Z0-9]"), Literal("")), "patterns_field")() // Define the corrected expected plan @@ -127,13 +115,8 @@ class PPLLogicalPlanPatternsTranslatorTestSuite val messageAttribute = UnresolvedAttribute("message") val noNumbersAttribute = UnresolvedAttribute("no_numbers") - val hostExpression = Alias( - RegExpReplace( - messageAttribute, - Literal( - "[0-9]"), - Literal("")), - "no_numbers")() + val hostExpression = + Alias(RegExpReplace(messageAttribute, Literal("[0-9]"), Literal("")), "no_numbers")() // Define the corrected expected plan val expectedPlan = Project( @@ -157,18 +140,16 @@ class PPLLogicalPlanPatternsTranslatorTestSuite val context = new CatalystPlanContext val logPlan = planTransformer.visit( - plan(pplParser, "source=apache | patterns new_field='no_numbers' pattern='[0-9]' message | top 1 no_numbers", false), + plan( + pplParser, + "source=apache | patterns new_field='no_numbers' pattern='[0-9]' message | top 1 no_numbers", + false), context) val messageAttribute = UnresolvedAttribute("message") val noNumbersAttribute = UnresolvedAttribute("no_numbers") - val hostExpression = Alias( - RegExpReplace( - messageAttribute, - Literal( - "[0-9]"), - Literal("")), - "no_numbers")() + val hostExpression = + Alias(RegExpReplace(messageAttribute, Literal("[0-9]"), Literal("")), "no_numbers")() val sortedPlan = Sort( Seq(