Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Temporarily support 4+ parts table identifier #913

Merged
merged 5 commits into from
Nov 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -541,11 +541,6 @@ class FlintSparkPPLBasicITSuite
| """.stripMargin))
assert(ex.getMessage().contains("TABLE_OR_VIEW_NOT_FOUND"))
}
val t7 = "spark_catalog.default.flint_ppl_test7.log"
val ex = intercept[IllegalArgumentException](sql(s"""
| source = $t7| head 2
| """.stripMargin))
assert(ex.getMessage().contains("Invalid table name"))
}

test("test describe backtick table names and name contains '.'") {
Expand All @@ -564,11 +559,6 @@ class FlintSparkPPLBasicITSuite
| """.stripMargin))
assert(ex.getMessage().contains("TABLE_OR_VIEW_NOT_FOUND"))
}
val t7 = "spark_catalog.default.flint_ppl_test7.log"
val ex = intercept[IllegalArgumentException](sql(s"""
| describe $t7
| """.stripMargin))
assert(ex.getMessage().contains("Invalid table name"))
}

test("test explain backtick table names and name contains '.'") {
Expand All @@ -590,12 +580,27 @@ class FlintSparkPPLBasicITSuite
Project(Seq(UnresolvedStar(None)), relation),
ExplainMode.fromString("extended"))
comparePlans(logicalPlan, expectedPlan, checkAnalysis = false)
}

// TODO Do not support 4+ parts table identifier in future (may be reverted this PR in 0.8.0)
test("test table name with more than 3 parts") {
val t7 = "spark_catalog.default.flint_ppl_test7.log"
val ex = intercept[IllegalArgumentException](sql(s"""
| explain extended | source = $t7
| """.stripMargin))
assert(ex.getMessage().contains("Invalid table name"))
val t4Parts = "`spark_catalog`.default.`startTime:1,endTime:2`.`this(is:['a/name'])`"
val t5Parts =
"`spark_catalog`.default.`startTime:1,endTime:2`.`this(is:['sub/name'])`.`this(is:['sub-sub/name'])`"
Seq(t7, t4Parts, t5Parts).foreach { table =>
val ex = intercept[AnalysisException](sql(s"""
| source = $table| head 2
| """.stripMargin))
assert(ex.getMessage().contains("TABLE_OR_VIEW_NOT_FOUND"))
}

Seq(t7, t4Parts, t5Parts).foreach { table =>
val ex = intercept[AnalysisException](sql(s"""
| describe $table
| """.stripMargin))
assert(ex.getMessage().contains("TABLE_OR_VIEW_NOT_FOUND"))
}
Comment on lines +591 to +603
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we test the TableIdentifier was built as expected?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added some UTs.

}

test("Search multiple tables - translated into union call with fields") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,15 @@ static TableIdentifier getTableIdentifier(QualifiedName qualifiedName) {
Option$.MODULE$.apply(qualifiedName.getParts().get(1)),
Option$.MODULE$.apply(qualifiedName.getParts().get(0)));
} else {
throw new IllegalArgumentException("Invalid table name: " + qualifiedName
+ " Syntax: [ database_name. ] table_name");
// TODO Do not support 4+ parts table identifier in future (may be reverted this PR in 0.8.0)
// qualifiedName.getParts().size() > 3
// A Spark TableIdentifier should only contain 3 parts: tableName, databaseName and catalogName.
// If the qualifiedName has more than 3 parts,
// we merge all parts from 3 to last parts into the tableName as one whole
identifier = new TableIdentifier(
String.join(".", qualifiedName.getParts().subList(2, qualifiedName.getParts().size())),
Option$.MODULE$.apply(qualifiedName.getParts().get(1)),
Option$.MODULE$.apply(qualifiedName.getParts().get(0)));
}
return identifier;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ class PPLLogicalPlanBasicQueriesTranslatorTestSuite
private val planTransformer = new CatalystQueryPlanVisitor()
private val pplParser = new PPLSyntaxParser()

test("test error describe clause") {
// TODO Do not support 4+ parts table identifier in future (may be reverted this PR in 0.8.0)
ignore("test error describe clause") {
val context = new CatalystPlanContext
val thrown = intercept[IllegalArgumentException] {
planTransformer.visit(plan(pplParser, "describe t.b.c.d"), context)
Expand All @@ -50,6 +51,66 @@ class PPLLogicalPlanBasicQueriesTranslatorTestSuite
comparePlans(expectedPlan, logPlan, false)
}

// TODO Do not support 4+ parts table identifier in future (may be reverted this PR in 0.8.0)
test("test describe with backticks and more then 3 parts") {
val context = new CatalystPlanContext
val logPlan =
planTransformer.visit(plan(pplParser, "describe `t`.b.`c.d`.`e.f`"), context)

val expectedPlan = DescribeTableCommand(
TableIdentifier("c.d.e.f", Option("b"), Option("t")),
Map.empty[String, String].empty,
isExtended = true,
output = DescribeRelation.getOutputAttrs)
comparePlans(expectedPlan, logPlan, false)
}

test("test read table with backticks and more then 3 parts") {
val context = new CatalystPlanContext
val logPlan =
planTransformer.visit(plan(pplParser, "source=`t`.b.`c.d`.`e.f`"), context)
val table = UnresolvedRelation(Seq("t", "b", "c.d.e.f"))
val expectedPlan = Project(Seq(UnresolvedStar(None)), table)
comparePlans(expectedPlan, logPlan, false)
}

test("test describe with complex backticks and more then 3 parts") {
val context = new CatalystPlanContext
val logPlan =
planTransformer.visit(
plan(
pplParser,
"describe `_Basic`.default.`startTime:0,endTime:1`.`logGroups(logGroupIdentifier:['hello/service_log'])`"),
context)

val expectedPlan = DescribeTableCommand(
TableIdentifier(
"startTime:0,endTime:1.logGroups(logGroupIdentifier:['hello/service_log'])",
Option("default"),
Option("_Basic")),
Map.empty[String, String].empty,
isExtended = true,
output = DescribeRelation.getOutputAttrs)
comparePlans(expectedPlan, logPlan, false)
}

test("test read complex table with backticks and more then 3 parts") {
val context = new CatalystPlanContext
val logPlan =
planTransformer.visit(
plan(
pplParser,
"source=`_Basic`.default.`startTime:0,endTime:1`.`logGroups(logGroupIdentifier:['hello/service_log'])`"),
context)
val table = UnresolvedRelation(
Seq(
"_Basic",
"default",
"startTime:0,endTime:1.logGroups(logGroupIdentifier:['hello/service_log'])"))
val expectedPlan = Project(Seq(UnresolvedStar(None)), table)
comparePlans(expectedPlan, logPlan, false)
}

test("test describe FQN table clause") {
val context = new CatalystPlanContext
val logPlan =
Expand Down
Loading