From 7355864e2cf8329188e69365415ab0bd718d0c1a Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Tue, 10 Sep 2024 15:47:52 +0200 Subject: [PATCH] [SPARK-49583][SQL] Define the error sub-condition `SECONDS_FRACTION` for invalid seconds fraction pattern ### What changes were proposed in this pull request? In the PR, I propose new sub-condition `SECONDS_FRACTION` of the error condition `INVALID_DATETIME_PATTERN` in case when datetime pattern doesn't contain proper seconds fraction of variable length. ### Why are the changes needed? To fix the failure on internal assert. This change should improve user experience with Spark SQL. Before the changes, Spark fails while parsing the datetime patterns like `\nSSSS\r`: ``` java.lang.AssertionError: assertion failed at scala.Predef$.assert(Predef.scala:264) at org.apache.spark.ErrorClassesJsonReader.getMessageTemplate(ErrorClassesJSONReader.scala:91) at org.apache.spark.ErrorClassesJsonReader.getErrorMessage(ErrorClassesJSONReader.scala:46) ``` ### Does this PR introduce _any_ user-facing change? Should not. Only if user's code depends on the format of error message. After changes, users get the error: ``` org.apache.spark.SparkIllegalArgumentException: [INVALID_DATETIME_PATTERN.SECONDS_FRACTION] Unrecognized datetime pattern: . Cannot detect a seconds fraction pattern of variable length. Please make sure the pattern contains 'S', and does not contain illegal characters. SQLSTATE: 22007 ``` ### How was this patch tested? By running new test suites: ``` $ build/sbt "test:testOnly *org.apache.spark.sql.catalyst.util.DateTimeFormatterHelperSuite" ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #48058 from MaxGekk/fix-INVALID_DATETIME_PATTERN. Authored-by: Max Gekk Signed-off-by: Max Gekk --- .../src/main/resources/error/error-conditions.json | 5 +++++ .../sql/catalyst/util/DateTimeFormatterHelper.scala | 2 +- .../catalyst/util/DateTimeFormatterHelperSuite.scala | 10 ++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 0affceab971fc..4bc48c042a0b3 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -2092,6 +2092,11 @@ "message" : [ "Too many letters in datetime pattern: . Please reduce pattern length." ] + }, + "SECONDS_FRACTION" : { + "message" : [ + "Cannot detect a seconds fraction pattern of variable length. Please make sure the pattern contains 'S', and does not contain illegal characters." + ] } }, "sqlState" : "22007" diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala index 343468dc0270c..71777906f868e 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala @@ -243,7 +243,7 @@ private object DateTimeFormatterHelper { rest = suffix case _ => throw new SparkIllegalArgumentException( - errorClass = "INVALID_DATETIME_PATTERN", + errorClass = "INVALID_DATETIME_PATTERN.SECONDS_FRACTION", messageParameters = Map("pattern" -> pattern)) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala index 034010f5825b8..6aa4f443c6add 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala @@ -83,4 +83,14 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite { assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss") assert(convertIncompatiblePattern("yyyy-MM-dd'T'") === "uuuu-MM-dd'T'") } + + test("SPARK-49583: invalid var length second fraction") { + val pattern = "\nSSSS\r" + checkError( + exception = intercept[SparkIllegalArgumentException] { + createBuilderWithVarLengthSecondFraction(pattern) + }, + errorClass = "INVALID_DATETIME_PATTERN.SECONDS_FRACTION", + parameters = Map("pattern" -> pattern)) + } }