Skip to content

Commit

Permalink
check case sensitivity flag and pick comparison methods according to …
Browse files Browse the repository at this point in the history
…the result in validateSchemaOutput
  • Loading branch information
averyqi-db committed Sep 16, 2024
1 parent 8d78f5b commit 3fbb1b0
Showing 1 changed file with 16 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.trees.TreePattern.{LOGICAL_QUERY_STAGE, Tre
import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.catalyst.util.MetadataColumnHelper
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
import org.apache.spark.sql.internal.SqlApiConf
import org.apache.spark.sql.types.{DataType, StructType}


Expand Down Expand Up @@ -416,12 +417,22 @@ object LogicalPlanIntegrity {
}

def validateSchemaOutput(previousPlan: LogicalPlan, currentPlan: LogicalPlan): Option[String] = {
if (!DataTypeUtils.equalsIgnoreNullability(previousPlan.schema, currentPlan.schema)) {
Some(s"The plan output schema has changed from ${previousPlan.schema.sql} to " +
currentPlan.schema.sql + s". The previous plan: ${previousPlan.treeString}\nThe new " +
"plan:\n" + currentPlan.treeString)
if (SqlApiConf.get.caseSensitiveAnalysis) {
if (!DataTypeUtils.equalsIgnoreNullability(previousPlan.schema, currentPlan.schema)) {
Some(s"The plan output schema has changed from ${previousPlan.schema.sql} to " +
currentPlan.schema.sql + s". The previous plan: ${previousPlan.treeString}\nThe new " +
"plan:\n" + currentPlan.treeString)
} else {
None
}
} else {
None
if (!DataTypeUtils.equalsIgnoreCaseAndNullability(previousPlan.schema, currentPlan.schema)) {
Some(s"The plan output schema has changed from ${previousPlan.schema.sql} to " +
currentPlan.schema.sql + s". The previous plan: ${previousPlan.treeString}\nThe new " +
"plan:\n" + currentPlan.treeString)
} else {
None
}
}
}

Expand Down

0 comments on commit 3fbb1b0

Please sign in to comment.