From 982a21fc6e4920b6703a85851ba1d81a66ffd304 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 12 Sep 2024 14:32:01 -0700 Subject: [PATCH] fix describe ppl command to be aware of 3 parts FQN table that includes schema.catalog.table Signed-off-by: YANGDB --- .../spark/ppl/FlintSparkPPLBasicITSuite.scala | 138 +++++++++--------- ...lPlanBasicQueriesTranslatorTestSuite.scala | 4 +- 2 files changed, 69 insertions(+), 73 deletions(-) diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBasicITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBasicITSuite.scala index d9b11697b..90ffe0385 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBasicITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBasicITSuite.scala @@ -43,83 +43,79 @@ class FlintSparkPPLBasicITSuite describe flint_ppl_test """.stripMargin) - // Retrieve the results - val results: Array[Row] = frame.collect() - // Define the expected results - val expectedResults: Array[Row] = Array( - Row("name", "string", null), - Row("age", "int", null), - Row("state", "string", null), - Row("country", "string", null), - Row("year", "int", null), - Row("month", "int", null), - Row("# Partition Information", "", ""), - Row("# col_name", "data_type", "comment"), - Row("year", "int", null), - Row("month", "int", null)) - - // Convert actual results to a Set for quick lookup - val resultsSet: Set[Row] = results.toSet - // Check that each expected row is present in the actual results - expectedResults.foreach { expectedRow => - assert( - resultsSet.contains(expectedRow), - s"Expected row $expectedRow not found in results") - } - // Retrieve the logical plan - val logicalPlan: LogicalPlan = - frame.queryExecution.commandExecuted.asInstanceOf[CommandResult].commandLogicalPlan - // Define the expected logical plan - val expectedPlan: LogicalPlan = - DescribeTableCommand( - TableIdentifier("flint_ppl_test"), - Map.empty[String, String], - isExtended = true, - output = DescribeRelation.getOutputAttrs) - // Compare the two plans - comparePlans(logicalPlan, expectedPlan, checkAnalysis = false) + // Retrieve the results + val results: Array[Row] = frame.collect() + // Define the expected results + val expectedResults: Array[Row] = Array( + Row("name", "string", null), + Row("age", "int", null), + Row("state", "string", null), + Row("country", "string", null), + Row("year", "int", null), + Row("month", "int", null), + Row("# Partition Information", "", ""), + Row("# col_name", "data_type", "comment"), + Row("year", "int", null), + Row("month", "int", null)) + + // Convert actual results to a Set for quick lookup + val resultsSet: Set[Row] = results.toSet + // Check that each expected row is present in the actual results + expectedResults.foreach { expectedRow => + assert(resultsSet.contains(expectedRow), s"Expected row $expectedRow not found in results") + } + // Retrieve the logical plan + val logicalPlan: LogicalPlan = + frame.queryExecution.commandExecuted.asInstanceOf[CommandResult].commandLogicalPlan + // Define the expected logical plan + val expectedPlan: LogicalPlan = + DescribeTableCommand( + TableIdentifier("flint_ppl_test"), + Map.empty[String, String], + isExtended = true, + output = DescribeRelation.getOutputAttrs) + // Compare the two plans + comparePlans(logicalPlan, expectedPlan, checkAnalysis = false) } test("describe (extended) FQN (2 parts) table query test") { - val frame = sql(s""" + val frame = sql(s""" describe default.flint_ppl_test """.stripMargin) - // Retrieve the results - val results: Array[Row] = frame.collect() - // Define the expected results - val expectedResults: Array[Row] = Array( - Row("name", "string", null), - Row("age", "int", null), - Row("state", "string", null), - Row("country", "string", null), - Row("year", "int", null), - Row("month", "int", null), - Row("# Partition Information", "", ""), - Row("# col_name", "data_type", "comment"), - Row("year", "int", null), - Row("month", "int", null)) - - // Convert actual results to a Set for quick lookup - val resultsSet: Set[Row] = results.toSet - // Check that each expected row is present in the actual results - expectedResults.foreach { expectedRow => - assert( - resultsSet.contains(expectedRow), - s"Expected row $expectedRow not found in results") - } - // Retrieve the logical plan - val logicalPlan: LogicalPlan = - frame.queryExecution.commandExecuted.asInstanceOf[CommandResult].commandLogicalPlan - // Define the expected logical plan - val expectedPlan: LogicalPlan = - DescribeTableCommand( - TableIdentifier("flint_ppl_test", Option("default")), - Map.empty[String, String], - isExtended = true, - output = DescribeRelation.getOutputAttrs) - // Compare the two plans - comparePlans(logicalPlan, expectedPlan, checkAnalysis = false) + // Retrieve the results + val results: Array[Row] = frame.collect() + // Define the expected results + val expectedResults: Array[Row] = Array( + Row("name", "string", null), + Row("age", "int", null), + Row("state", "string", null), + Row("country", "string", null), + Row("year", "int", null), + Row("month", "int", null), + Row("# Partition Information", "", ""), + Row("# col_name", "data_type", "comment"), + Row("year", "int", null), + Row("month", "int", null)) + + // Convert actual results to a Set for quick lookup + val resultsSet: Set[Row] = results.toSet + // Check that each expected row is present in the actual results + expectedResults.foreach { expectedRow => + assert(resultsSet.contains(expectedRow), s"Expected row $expectedRow not found in results") + } + // Retrieve the logical plan + val logicalPlan: LogicalPlan = + frame.queryExecution.commandExecuted.asInstanceOf[CommandResult].commandLogicalPlan + // Define the expected logical plan + val expectedPlan: LogicalPlan = + DescribeTableCommand( + TableIdentifier("flint_ppl_test", Option("default")), + Map.empty[String, String], + isExtended = true, + output = DescribeRelation.getOutputAttrs) + // Compare the two plans + comparePlans(logicalPlan, expectedPlan, checkAnalysis = false) } test("create ppl simple query test") { diff --git a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanBasicQueriesTranslatorTestSuite.scala b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanBasicQueriesTranslatorTestSuite.scala index fea1bf041..ad3ee18ba 100644 --- a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanBasicQueriesTranslatorTestSuite.scala +++ b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanBasicQueriesTranslatorTestSuite.scala @@ -36,10 +36,10 @@ class PPLLogicalPlanBasicQueriesTranslatorTestSuite thrown.getMessage === "Invalid table name: t.b.c.d Syntax: [ database_name. ] table_name") } - test("test describe FQN table clause") { val context = new CatalystPlanContext - val logPlan = planTransformer.visit(plan(pplParser, "describe schema.default.http_logs", false), context) + val logPlan = + planTransformer.visit(plan(pplParser, "describe schema.default.http_logs", false), context) val expectedPlan = DescribeTableCommand( TableIdentifier("http_logs", Option("schema"), Option("default")),