From 915b12fde883d9423093bd26f56ede13454c4e15 Mon Sep 17 00:00:00 2001 From: Yuyang Wang Date: Wed, 29 May 2024 16:59:11 -0700 Subject: [PATCH 1/4] fix and add test --- .../snowflake/snowpark/types/Variant.scala | 6 ++++- .../snowpark_test/DataTypeSuite.scala | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/snowflake/snowpark/types/Variant.scala b/src/main/scala/com/snowflake/snowpark/types/Variant.scala index 6ca52a12..66d2b737 100644 --- a/src/main/scala/com/snowflake/snowpark/types/Variant.scala +++ b/src/main/scala/com/snowflake/snowpark/types/Variant.scala @@ -164,7 +164,11 @@ class Variant private[snowpark] ( def this(str: String) = this({ try { - MAPPER.readTree(str) + if (str.toLowerCase().startsWith("null") && str != "null") { + JsonNodeFactory.instance.textNode(str) + } else { + MAPPER.readTree(str) + } } catch { case _: Exception => JsonNodeFactory.instance.textNode(str) } diff --git a/src/test/scala/com/snowflake/snowpark_test/DataTypeSuite.scala b/src/test/scala/com/snowflake/snowpark_test/DataTypeSuite.scala index 304dbbf5..0d19d5e8 100644 --- a/src/test/scala/com/snowflake/snowpark_test/DataTypeSuite.scala +++ b/src/test/scala/com/snowflake/snowpark_test/DataTypeSuite.scala @@ -3,6 +3,7 @@ package com.snowflake.snowpark_test import com.snowflake.snowpark.{Row, SNTestBase, TestUtils} import com.snowflake.snowpark.types._ import com.snowflake.snowpark.functions._ +import com.snowflake.snowpark.internal.Utils import java.sql.{Date, Time, Timestamp} import java.util.TimeZone @@ -588,4 +589,28 @@ class DataTypeSuite extends SNTestBase { |""".stripMargin) // scalastyle:on } + + test("Variant containing word null in the text") { + import session.implicits._ + var variant = new Variant("null string starts with null") + var df = Seq(variant).toDF("a") + checkAnswer( + df, + Row("\"null string starts with null\"") + ) + + variant = new Variant("string with null in the middle") + df = Seq(variant).toDF("a") + checkAnswer( + df, + Row("\"string with null in the middle\"") + ) + + variant = new Variant("string with null in the end null") + df = Seq(variant).toDF("a") + checkAnswer( + df, + Row("\"string with null in the end null\"") + ) + } } From 3ca408b36b8c3ea3c2d6a2ab4ce481196f99a11b Mon Sep 17 00:00:00 2001 From: Yuyang Wang Date: Tue, 4 Jun 2024 10:25:05 -0700 Subject: [PATCH 2/4] add comment --- src/main/scala/com/snowflake/snowpark/types/Variant.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/scala/com/snowflake/snowpark/types/Variant.scala b/src/main/scala/com/snowflake/snowpark/types/Variant.scala index 66d2b737..7d776e44 100644 --- a/src/main/scala/com/snowflake/snowpark/types/Variant.scala +++ b/src/main/scala/com/snowflake/snowpark/types/Variant.scala @@ -164,6 +164,9 @@ class Variant private[snowpark] ( def this(str: String) = this({ try { + // `ObjectMapper` only reads the first token from the input string but not the whole string. + // For example, it can successfully convert "null dummy" to + // `null` value without reporting error. if (str.toLowerCase().startsWith("null") && str != "null") { JsonNodeFactory.instance.textNode(str) } else { From 8926dfbaeb7ae6ce6174cd60b5282f4417846de0 Mon Sep 17 00:00:00 2001 From: Yuyang Wang Date: Tue, 4 Jun 2024 10:32:17 -0700 Subject: [PATCH 3/4] update comment --- src/main/scala/com/snowflake/snowpark/types/Variant.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/scala/com/snowflake/snowpark/types/Variant.scala b/src/main/scala/com/snowflake/snowpark/types/Variant.scala index 7d776e44..472c8ffb 100644 --- a/src/main/scala/com/snowflake/snowpark/types/Variant.scala +++ b/src/main/scala/com/snowflake/snowpark/types/Variant.scala @@ -165,8 +165,7 @@ class Variant private[snowpark] ( this({ try { // `ObjectMapper` only reads the first token from the input string but not the whole string. - // For example, it can successfully convert "null dummy" to - // `null` value without reporting error. + // It can successfully convert "null dummy" to `null` value without reporting error. if (str.toLowerCase().startsWith("null") && str != "null") { JsonNodeFactory.instance.textNode(str) } else { From f54fce19d4522ad9a800e1f96d3d537ab0d04a11 Mon Sep 17 00:00:00 2001 From: Yuyang Wang Date: Tue, 4 Jun 2024 10:48:00 -0700 Subject: [PATCH 4/4] fix comment --- .pre-commit-config.yaml | 2 +- src/main/scala/com/snowflake/snowpark/types/Variant.scala | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e9aa7be..c2af46b4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: - repo: git@github.com:snowflakedb/casec_precommit.git - rev: HEAD + rev: v1.35.4 hooks: - id: secret-scanner diff --git a/src/main/scala/com/snowflake/snowpark/types/Variant.scala b/src/main/scala/com/snowflake/snowpark/types/Variant.scala index 472c8ffb..28ce5b67 100644 --- a/src/main/scala/com/snowflake/snowpark/types/Variant.scala +++ b/src/main/scala/com/snowflake/snowpark/types/Variant.scala @@ -164,8 +164,10 @@ class Variant private[snowpark] ( def this(str: String) = this({ try { - // `ObjectMapper` only reads the first token from the input string but not the whole string. - // It can successfully convert "null dummy" to `null` value without reporting error. + // `ObjectMapper` only reads the first token from + // the input string but not the whole string. + // For example, It can successfully + // convert "null dummy" to `null` value without reporting error. if (str.toLowerCase().startsWith("null") && str != "null") { JsonNodeFactory.instance.textNode(str) } else {