From efacc14e058747cfa49c72f25dcddf2570b522fa Mon Sep 17 00:00:00 2001 From: Bing Li Date: Tue, 21 Nov 2023 10:49:23 -0800 Subject: [PATCH 1/2] remove usage of ParseUtils --- .../snowflake/snowpark/internal/UDFClassPath.scala | 13 ++----------- .../snowflake/snowpark/UDFRegistrationSuite.scala | 6 ------ 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/src/main/scala/com/snowflake/snowpark/internal/UDFClassPath.scala b/src/main/scala/com/snowflake/snowpark/internal/UDFClassPath.scala index bafe7a0e..aedbc8a6 100644 --- a/src/main/scala/com/snowflake/snowpark/internal/UDFClassPath.scala +++ b/src/main/scala/com/snowflake/snowpark/internal/UDFClassPath.scala @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.JsonNode import java.io.File import java.net.{URI, URLClassLoader} import com.snowflake.snowpark.Session -import sun.net.www.ParseUtil object UDFClassPath extends Logging { @@ -94,13 +93,7 @@ object UDFClassPath extends Logging { if (path.contains(":")) { path = path.substring(path.indexOf(":") + 1) } - /* - * The URL in class loader is encoded, so we have to decode it to read the - * local file. - */ - path = ParseUtil.decode(path) - // Creating a File fixes the separator based on OS - new File(path).getPath + new URI(path).getPath } } @@ -111,9 +104,7 @@ object UDFClassPath extends Logging { * The URL in CodeSource location is encoded, so we have to decode it to read the * local file. */ - val parsed = ParseUtil.decode(codeSource.getLocation.getPath) - // Creating a File fixes the separator based on OS - Some(new File(parsed).getPath) + Some(new URI(codeSource.getLocation.getPath).getPath) } else { None } diff --git a/src/test/scala/com/snowflake/snowpark/UDFRegistrationSuite.scala b/src/test/scala/com/snowflake/snowpark/UDFRegistrationSuite.scala index b18207be..1c0984b6 100644 --- a/src/test/scala/com/snowflake/snowpark/UDFRegistrationSuite.scala +++ b/src/test/scala/com/snowflake/snowpark/UDFRegistrationSuite.scala @@ -5,7 +5,6 @@ import com.snowflake.snowpark.internal.Utils.clientPackageName import java.io.{BufferedOutputStream, File, FileOutputStream} import java.nio.file.{Files, NoSuchFileException} import com.snowflake.snowpark.internal.{JavaUtils, UDFClassPath} -import sun.net.www.ParseUtil import scala.reflect.internal.util.BatchSourceFile import scala.reflect.io.{AbstractFile, VirtualDirectory} @@ -23,11 +22,6 @@ class UDFRegistrationSuite extends SNTestBase with FileUtils { session.runQuery(s"create or replace temporary stage $tempStage") } - test("Test URL encoding") { - val inputs = Seq("dbiufwhronr==", "fdeswfirn--", "bsdij++", "sb#i", "bxsj%i@9h(nb)") - inputs.foreach(str => assert(ParseUtil.decode(ParseUtil.encodePath(str)) == str)) - } - test("Test that jar files are uploaded to stage correctly") { val udfRegistrar = new UDFRegistration(session) val classDirs = UDFClassPath.classDirs(session).toList From 01507c741b2c0016cdfc1da803e3fa7cf04fcce9 Mon Sep 17 00:00:00 2001 From: Bing Li Date: Tue, 21 Nov 2023 17:06:52 -0800 Subject: [PATCH 2/2] fix windows tests --- .../snowpark/UDFClasspathSuite.scala | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/test/scala/com/snowflake/snowpark/UDFClasspathSuite.scala b/src/test/scala/com/snowflake/snowpark/UDFClasspathSuite.scala index 33d67434..3c3f388d 100644 --- a/src/test/scala/com/snowflake/snowpark/UDFClasspathSuite.scala +++ b/src/test/scala/com/snowflake/snowpark/UDFClasspathSuite.scala @@ -110,19 +110,7 @@ class UDFClasspathSuite extends SNTestBase { test("Test for getPathForClass") { val clazz = classOf[scala.StringBuilder] - val expectedUrl = if (Utils.isWindows) { - // On windows, the directory format is gotten from getCodeSource() is like: - // /C:/Users/runneradmin/.../scala-library-2.12.11.jar - // The directory gotten with UDFClassPath.getPathForClass(clazz) is like: - // C:\Users\runneradmin\...\scala-library-2.12.11.jar - // So, need format conversion for result comparison. - clazz.getProtectionDomain.getCodeSource.getLocation.getPath - .substring(1) - .replace("/", "\\") - } else { - clazz.getProtectionDomain.getCodeSource.getLocation.getPath - } - + val expectedUrl = clazz.getProtectionDomain.getCodeSource.getLocation.getPath val result = UDFClassPath.getPathForClass(clazz) assert(result.isDefined && result.get.equals(expectedUrl)) } @@ -138,7 +126,18 @@ class UDFClasspathSuite extends SNTestBase { val foundJarPath = UDFClassPath.getPathUsingClassLoader(classToLoad) assert(foundJarPath.nonEmpty) assert(new File(foundJarPath.get).exists()) - assert(foundJarPath.get.equals(jarFilePath)) + + val expectedUrl = if (Utils.isWindows) { + // On windows, the directory format is gotten from getCodeSource() is like: + // /C:/Users/runneradmin/.../scala-library-2.12.11.jar + // The directory gotten with UDFClassPath.getPathForClass(clazz) is like: + // C:\Users\runneradmin\...\scala-library-2.12.11.jar + // So, need format conversion for result comparison. + "/" + jarFilePath.replace("\\", "/") + } else { + jarFilePath + } + assert(foundJarPath.get.equals(expectedUrl)) assert(foundJarPath.get.contains(prefix)) }