From 312bc3621ae7079bad79c7047f76f370fb1aacac Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Tue, 26 Sep 2023 09:27:27 -0400 Subject: [PATCH 1/3] Fixes #343 by removing DELETE translation rule --- inst/csv/replacementPatterns.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index 9b61aaef..1691c1b5 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -1130,7 +1130,6 @@ spark,(SELECT DISTINCT TOP @([0-9]+)rows @a),(SELECT DISTINCT @a LIMIT @rows) spark,"WITH @cte AS (SELECT @s1 '@literal' @s2)","WITH @cte AS (SELECT @s1 CAST('@literal' as STRING) @s2)" spark,UPDATE STATISTICS @a;, spark,"SELECT @columns FROM (@a) @x,(@b) @y;","SELECT @columns FROM (@a) @x cross join (@b) @y;" -spark,DELETE FROM @a WHERE @b;,INSERT OVERWRITE TABLE @a SELECT * FROM @a WHERE NOT (@b); sqlite extended,"IIF(@condition, @whentrue, @whenfalse)","CASE WHEN @condition THEN @whentrue ELSE @whenfalse END" sqlite extended,"ROUND(@a,@b)","ROUND(CAST(@a AS REAL),@b)" sqlite extended,+ '@a',|| '@a' From 9aaef1a1bf3c8f61ec91fec24f0882dc7b92522a Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Tue, 26 Sep 2023 09:29:33 -0400 Subject: [PATCH 2/3] Remove INSERT INTO replacement rules for spark --- inst/csv/replacementPatterns.csv | 2 -- 1 file changed, 2 deletions(-) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index 1691c1b5..f9f2456e 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -1039,8 +1039,6 @@ spark,#@([^\s]+)table.@([^\s]+)field,%session_id%@table.@field spark,#,%temp_prefix%%session_id% spark,"--HINT BUCKET(@a, @b)", spark,"--HINT PARTITION(@a @b)", -spark,"INSERT INTO @table (@columns) SELECT @values;","WITH insertion_temp AS (\n(SELECT @values) UNION ALL (SELECT @columns FROM @table))\nINSERT OVERWRITE TABLE @table (@columns) SELECT * FROM insertion_temp;" -spark,"INSERT INTO @table (@columns) VALUES @values;","WITH insertion_temp AS (\n(SELECT * FROM VALUES @values T(@columns)) UNION ALL (SELECT @columns FROM @table))\nINSERT OVERWRITE TABLE @table (@columns) SELECT * FROM insertion_temp;" spark,"HINT DISTRIBUTE_ON_KEY(@key) CREATE TABLE IF NOT EXISTS @table\nUSING DELTA\nAS\n@definition;","HINT DISTRIBUTE_ON_KEY(@key)\nCREATE TABLE IF NOT EXISTS @table\nUSING DELTA\nAS\n@definition;\nOPTIMIZE @table ZORDER BY @key;" spark,"HINT DISTRIBUTE_ON_KEY(@key) IF OBJECT_ID('@d', 'U') IS NULL WITH @a AS @b SELECT @c INTO @d FROM @e;",HINT DISTRIBUTE_ON_KEY(@key)\nCREATE TABLE IF NOT EXISTS @d\nUSING DELTA\nAS\nWITH @a AS @b SELECT\n@c\nFROM\n@e;\nOPTIMIZE @d ZORDER BY @key; spark,"HINT DISTRIBUTE_ON_KEY(@key) IF OBJECT_ID('@b', 'U') IS NULL SELECT @a INTO @b FROM @c;",HINT DISTRIBUTE_ON_KEY(@key) \nCREATE TABLE IF NOT EXISTS @b\nUSING DELTA\nAS\nSELECT\n@a\nFROM\n@c;\nOPTIMIZE @b ZORDER BY @key; From c806a474a29faab99be2601ddf80c1bc1050d206 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Tue, 26 Sep 2023 09:42:56 -0400 Subject: [PATCH 3/3] Remove unit tests associated with spark translation rules that were removed --- tests/testthat/test-translate-spark.R | 29 --------------------------- 1 file changed, 29 deletions(-) diff --git a/tests/testthat/test-translate-spark.R b/tests/testthat/test-translate-spark.R index bb51f170..76daeb1a 100644 --- a/tests/testthat/test-translate-spark.R +++ b/tests/testthat/test-translate-spark.R @@ -376,35 +376,6 @@ test_that("translate sql server -> spark DROP TABLE IF EXISTS", { expect_equal_ignore_spaces(sql, "DROP TABLE IF EXISTS test;") }) -test_that("translate sql server -> spark INSERT INTO SELECT", { - sql <- translate("INSERT INTO a (b int) SELECT c FROM cte1;", - targetDialect = "spark" - ) - expect_equal_ignore_spaces( - sql, - "WITH insertion_temp AS (\n(SELECT c FROM cte1) UNION ALL (SELECT b int FROM a ))\nINSERT OVERWRITE TABLE a (b int) SELECT * FROM insertion_temp;" - ) -}) - -test_that("translate sql server -> spark INSERT INTO VALUES", { - sql <- translate("INSERT INTO my_table (key,value) VALUES (1,0),(2,0),(3,1);", - targetDialect = "spark" - ) - expect_equal_ignore_spaces( - sql, - "WITH insertion_temp AS (\n(SELECT * FROM VALUES (1,0),(2,0),(3,1) T(key,value)) UNION ALL (SELECT key,value FROM my_table ))\nINSERT OVERWRITE TABLE my_table (key,value) SELECT * FROM insertion_temp;" - ) -}) - -test_that("translate sql server -> spark DELETE FROM WHERE", { - sql <- translate("DELETE FROM my_table WHERE a=1;", - targetDialect = "spark" - ) - expect_equal_ignore_spaces( - sql, - "INSERT OVERWRITE TABLE my_table SELECT * FROM my_table WHERE NOT (a=1);" - ) -}) test_that("translate sql server -> spark double CTE INSERT INTO", { sql <- translate( "WITH a AS (