From 3fbb1b0744911b51c6f1050c870e4a402ebedfcb Mon Sep 17 00:00:00 2001 From: Avery Qi Date: Mon, 16 Sep 2024 14:03:13 -0700 Subject: [PATCH] check case sensitivity flag and pick comparison methods according to the result in validateSchemaOutput --- .../catalyst/plans/logical/LogicalPlan.scala | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala index 5563199205448..5a30aef3b0e48 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.trees.TreePattern.{LOGICAL_QUERY_STAGE, Tre import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.MetadataColumnHelper import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} +import org.apache.spark.sql.internal.SqlApiConf import org.apache.spark.sql.types.{DataType, StructType} @@ -416,12 +417,22 @@ object LogicalPlanIntegrity { } def validateSchemaOutput(previousPlan: LogicalPlan, currentPlan: LogicalPlan): Option[String] = { - if (!DataTypeUtils.equalsIgnoreNullability(previousPlan.schema, currentPlan.schema)) { - Some(s"The plan output schema has changed from ${previousPlan.schema.sql} to " + - currentPlan.schema.sql + s". The previous plan: ${previousPlan.treeString}\nThe new " + - "plan:\n" + currentPlan.treeString) + if (SqlApiConf.get.caseSensitiveAnalysis) { + if (!DataTypeUtils.equalsIgnoreNullability(previousPlan.schema, currentPlan.schema)) { + Some(s"The plan output schema has changed from ${previousPlan.schema.sql} to " + + currentPlan.schema.sql + s". The previous plan: ${previousPlan.treeString}\nThe new " + + "plan:\n" + currentPlan.treeString) + } else { + None + } } else { - None + if (!DataTypeUtils.equalsIgnoreCaseAndNullability(previousPlan.schema, currentPlan.schema)) { + Some(s"The plan output schema has changed from ${previousPlan.schema.sql} to " + + currentPlan.schema.sql + s". The previous plan: ${previousPlan.treeString}\nThe new " + + "plan:\n" + currentPlan.treeString) + } else { + None + } } }