From f8e942af55951907515ef7bd50550ae03f7cadac Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 2 Oct 2023 11:00:38 -0400 Subject: [PATCH 1/9] Added replacement patterns and tests for datediff() in Spark --- inst/csv/replacementPatterns.csv | 9 ++++++ tests/testthat/test-translate-spark.R | 46 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index 90094cbb..ca8818d6 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -1076,9 +1076,18 @@ spark,"DATEADD(yyyy,@years,@date)",(@date + INTERVAL @years year) spark,"DATEADD(year,@years,@date)",(@date + INTERVAL @years year) spark,"date_add(@date, @(-?[0-9]+)a.0)","date_add(@date, @a)" spark,INTERVAL @(-?[0-9]+)a.0,INTERVAL @a +spark,"DATEDIFF(second,@start, @end)","datediff(second,@end,@start)" +spark,"DATEDIFF(minute,@start, @end)","datediff(minute,@end,@start)" +spark,"DATEDIFF(hour,@start, @end)","datediff(hour,@end,@start)" spark,"DATEDIFF(d,@start, @end)","datediff(@end,@start)" spark,"DATEDIFF(dd,@start, @end)","datediff(@end,@start)" spark,"DATEDIFF(day,@start, @end)","datediff(@end,@start)" +spark,"DATEDIFF(m,@start, @end)","datediff(month,@end,@start)" +spark,"DATEDIFF(mm,@start, @end)","datediff(month,@end,@start)" +spark,"DATEDIFF(month,@start, @end)","datediff(month,@end,@start)" +spark,"DATEDIFF(yy,@start, @end)","datediff(year,@end,@start)" +spark,"DATEDIFF(yyyy,@start, @end)","datediff(year,@end,@start)" +spark,"DATEDIFF(year,@start, @end)","datediff(year,@end,@start)" spark,"CONVERT(VARCHAR,@date,112)","@date" spark,GETDATE(),CURRENT_DATE spark,"+ LEFT(@str,@chars)","SUBSTR(@str,@chars)" diff --git a/tests/testthat/test-translate-spark.R b/tests/testthat/test-translate-spark.R index bb51f170..ae096f08 100644 --- a/tests/testthat/test-translate-spark.R +++ b/tests/testthat/test-translate-spark.R @@ -132,6 +132,52 @@ test_that("translate sql server -> spark datediff", { targetDialect = "spark" ) expect_equal_ignore_spaces(sql, "SELECT datediff('2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(second, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(second, '2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(minute, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(minute, '2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(hour, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(hour, '2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(m, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(month, '2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(mm, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(month, '2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(month, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(month, '2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(yy, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(year, '2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(year, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(year, '2019-01-02', '2019-01-01')") + + sql <- translate("SELECT datediff(yyyy, '2019-01-01', '2019-01-02')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT datediff(year, '2019-01-02', '2019-01-01')") + }) test_that("translate sql server -> spark convert date", { From 47167ad2b0ec1f318aa9ce2fd9bea1ff092eb25a Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 2 Oct 2023 11:20:56 -0400 Subject: [PATCH 2/9] Now passes tests --- inst/csv/replacementPatterns.csv | 20 ++++++++------------ tests/testthat/test-translate-spark.R | 20 -------------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index ca8818d6..ce492291 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -1076,18 +1076,14 @@ spark,"DATEADD(yyyy,@years,@date)",(@date + INTERVAL @years year) spark,"DATEADD(year,@years,@date)",(@date + INTERVAL @years year) spark,"date_add(@date, @(-?[0-9]+)a.0)","date_add(@date, @a)" spark,INTERVAL @(-?[0-9]+)a.0,INTERVAL @a -spark,"DATEDIFF(second,@start, @end)","datediff(second,@end,@start)" -spark,"DATEDIFF(minute,@start, @end)","datediff(minute,@end,@start)" -spark,"DATEDIFF(hour,@start, @end)","datediff(hour,@end,@start)" -spark,"DATEDIFF(d,@start, @end)","datediff(@end,@start)" -spark,"DATEDIFF(dd,@start, @end)","datediff(@end,@start)" -spark,"DATEDIFF(day,@start, @end)","datediff(@end,@start)" -spark,"DATEDIFF(m,@start, @end)","datediff(month,@end,@start)" -spark,"DATEDIFF(mm,@start, @end)","datediff(month,@end,@start)" -spark,"DATEDIFF(month,@start, @end)","datediff(month,@end,@start)" -spark,"DATEDIFF(yy,@start, @end)","datediff(year,@end,@start)" -spark,"DATEDIFF(yyyy,@start, @end)","datediff(year,@end,@start)" -spark,"DATEDIFF(year,@start, @end)","datediff(year,@end,@start)" +spark,"DATEDIFF(second,@start,@end)","datediff(second,@end,@start)" +spark,"DATEDIFF(minute,@start,@end)","datediff(minute,@end,@start)" +spark,"DATEDIFF(hour,@start,@end)","datediff(hour,@end,@start)" +spark,"DATEDIFF(d,@start,@end)","datediff(@end,@start)" +spark,"DATEDIFF(dd,@start,@end)","datediff(@end,@start)" +spark,"DATEDIFF(day,@start,@end)","datediff(@end,@start)" +spark,"DATEDIFF(month,@start,@end)","datediff(month,@end,@start)" +spark,"DATEDIFF(year,@start,@end)","datediff(year,@end,@start)" spark,"CONVERT(VARCHAR,@date,112)","@date" spark,GETDATE(),CURRENT_DATE spark,"+ LEFT(@str,@chars)","SUBSTR(@str,@chars)" diff --git a/tests/testthat/test-translate-spark.R b/tests/testthat/test-translate-spark.R index ae096f08..77bd5e6e 100644 --- a/tests/testthat/test-translate-spark.R +++ b/tests/testthat/test-translate-spark.R @@ -148,36 +148,16 @@ test_that("translate sql server -> spark datediff", { ) expect_equal_ignore_spaces(sql, "SELECT datediff(hour, '2019-01-02', '2019-01-01')") - sql <- translate("SELECT datediff(m, '2019-01-01', '2019-01-02')", - targetDialect = "spark" - ) - expect_equal_ignore_spaces(sql, "SELECT datediff(month, '2019-01-02', '2019-01-01')") - - sql <- translate("SELECT datediff(mm, '2019-01-01', '2019-01-02')", - targetDialect = "spark" - ) - expect_equal_ignore_spaces(sql, "SELECT datediff(month, '2019-01-02', '2019-01-01')") - sql <- translate("SELECT datediff(month, '2019-01-01', '2019-01-02')", targetDialect = "spark" ) expect_equal_ignore_spaces(sql, "SELECT datediff(month, '2019-01-02', '2019-01-01')") - sql <- translate("SELECT datediff(yy, '2019-01-01', '2019-01-02')", - targetDialect = "spark" - ) - expect_equal_ignore_spaces(sql, "SELECT datediff(year, '2019-01-02', '2019-01-01')") - sql <- translate("SELECT datediff(year, '2019-01-01', '2019-01-02')", targetDialect = "spark" ) expect_equal_ignore_spaces(sql, "SELECT datediff(year, '2019-01-02', '2019-01-01')") - sql <- translate("SELECT datediff(yyyy, '2019-01-01', '2019-01-02')", - targetDialect = "spark" - ) - expect_equal_ignore_spaces(sql, "SELECT datediff(year, '2019-01-02', '2019-01-01')") - }) test_that("translate sql server -> spark convert date", { From 9fe1f54063c5181b8456ba66bbe2995c7fad12f0 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 2 Oct 2023 11:23:24 -0400 Subject: [PATCH 3/9] Revert datediff mappings for day --- inst/csv/replacementPatterns.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index ce492291..cdd01a1f 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -1079,9 +1079,9 @@ spark,INTERVAL @(-?[0-9]+)a.0,INTERVAL @a spark,"DATEDIFF(second,@start,@end)","datediff(second,@end,@start)" spark,"DATEDIFF(minute,@start,@end)","datediff(minute,@end,@start)" spark,"DATEDIFF(hour,@start,@end)","datediff(hour,@end,@start)" -spark,"DATEDIFF(d,@start,@end)","datediff(@end,@start)" -spark,"DATEDIFF(dd,@start,@end)","datediff(@end,@start)" -spark,"DATEDIFF(day,@start,@end)","datediff(@end,@start)" +spark,"DATEDIFF(d,@start, @end)","datediff(@end,@start)" +spark,"DATEDIFF(dd,@start, @end)","datediff(@end,@start)" +spark,"DATEDIFF(day,@start, @end)","datediff(@end,@start)" spark,"DATEDIFF(month,@start,@end)","datediff(month,@end,@start)" spark,"DATEDIFF(year,@start,@end)","datediff(year,@end,@start)" spark,"CONVERT(VARCHAR,@date,112)","@date" From 587eae42e788d0684943eede04c67c12318813f5 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 2 Oct 2023 12:19:29 -0400 Subject: [PATCH 4/9] Added datediff translations for Oracle --- inst/csv/replacementPatterns.csv | 3 +++ tests/testthat/test-translate-oracle.R | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index cdd01a1f..30b0207a 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -32,6 +32,9 @@ oracle,"DATEADD(m,@months,@date)","ADD_MONTHS(@date, @months)" oracle,"DATEADD(year,@years,@date)","ADD_MONTHS(@date, 12 * @years)" oracle,"DATEADD(yyyy,@years,@date)","ADD_MONTHS(@date, 12 * @years)" oracle,"DATEADD(yy,@years,@date)","ADD_MONTHS(@date, 12 * @years)" +oracle,"DATEDIFF(second,@start, @end)","EXTRACT(SECOND FROM (@end - @start))" +oracle,"DATEDIFF(minute,@start, @end)","EXTRACT(MINUTE FROM (@end - @start))" +oracle,"DATEDIFF(hour,@start, @end)","EXTRACT(HOUR FROM (@end - @start))" oracle,"DATEDIFF(day,@start, @end)",CEIL(CAST(@end AS DATE) - CAST(@start AS DATE)) oracle,"DATEDIFF(dd,@start, @end)",CEIL(CAST(@end AS DATE) - CAST(@start AS DATE)) oracle,"DATEDIFF(d,@start, @end)",CEIL(CAST(@end AS DATE) - CAST(@start AS DATE)) diff --git a/tests/testthat/test-translate-oracle.R b/tests/testthat/test-translate-oracle.R index 11ade8ae..ebcc9597 100644 --- a/tests/testthat/test-translate-oracle.R +++ b/tests/testthat/test-translate-oracle.R @@ -24,6 +24,30 @@ test_that("translate sql server -> Oracle DATEDIFF", { sql, "SELECT CEIL(CAST(drug_era_end_date AS DATE) - CAST(drug_era_start_date AS DATE)) FROM drug_era;" ) + + sql <- translate("SELECT DATEDIFF(second,drug_era_start_date,drug_era_end_date) FROM drug_era;", + targetDialect = "oracle" + ) + expect_equal_ignore_spaces( + sql, + "SELECT EXTRACT(SECOND FROM (drug_era_end_date - drug_era_start_date)) FROM drug_era;" + ) + + sql <- translate("SELECT DATEDIFF(minute,drug_era_start_date,drug_era_end_date) FROM drug_era;", + targetDialect = "oracle" + ) + expect_equal_ignore_spaces( + sql, + "SELECT EXTRACT(MINUTE FROM (drug_era_end_date - drug_era_start_date)) FROM drug_era;" + ) + + sql <- translate("SELECT DATEDIFF(hour,drug_era_start_date,drug_era_end_date) FROM drug_era;", + targetDialect = "oracle" + ) + expect_equal_ignore_spaces( + sql, + "SELECT EXTRACT(HOUR FROM (drug_era_end_date - drug_era_start_date)) FROM drug_era;" + ) }) test_that("translate sql server -> Oracle DATEDIFF year", { From bef6747d70010d4be26034f3e2f83c5f0af79896 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 2 Oct 2023 14:39:36 -0400 Subject: [PATCH 5/9] Add datediff translations for postGres --- inst/csv/replacementPatterns.csv | 3 +++ tests/testthat/test-translate-postgresql.R | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index 30b0207a..1ab848a7 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -174,6 +174,9 @@ postgresql,"DATEADD(month,@months,@date)",(@date + @months*INTERVAL'1 month') postgresql,"DATEADD(yy,@years,@date)",(@date + @years*INTERVAL'1 year') postgresql,"DATEADD(yyyy,@years,@date)",(@date + @years*INTERVAL'1 year') postgresql,"DATEADD(year,@years,@date)",(@date + @years*INTERVAL'1 year') +postgresql,"DATEDIFF(second,@start, @end)",EXTRACT(EPOCH FROM (@end - @start)) +postgresql,"DATEDIFF(minute,@start, @end)",(EXTRACT(EPOCH FROM (@end - @start)) / 60) +postgresql,"DATEDIFF(hour,@start, @end)",(EXTRACT(EPOCH FROM (@end - @start)) / 3600) postgresql,"DATEDIFF(d,@start, @end)",(CAST(@end AS DATE) - CAST(@start AS DATE)) postgresql,"DATEDIFF(dd,@start, @end)",(CAST(@end AS DATE) - CAST(@start AS DATE)) postgresql,"DATEDIFF(day,@start, @end)",(CAST(@end AS DATE) - CAST(@start AS DATE)) diff --git a/tests/testthat/test-translate-postgresql.R b/tests/testthat/test-translate-postgresql.R index 9bcdc542..efa86019 100644 --- a/tests/testthat/test-translate-postgresql.R +++ b/tests/testthat/test-translate-postgresql.R @@ -49,6 +49,23 @@ test_that("translate sql server -> PostgreSQL date diff (month)", { expect_equal_ignore_spaces(sql, "SELECT (extract(year from age(CAST(drug_era_end_date AS DATE), CAST(drug_era_start_date AS DATE)))*12 + extract(month from age(CAST(drug_era_end_date AS DATE), CAST(drug_era_start_date AS DATE)))) FROM drug_era;") }) +test_that("translate sql server -> PostgreSQL date diff (hour, minute, second)", { + sql <- translate("SELECT DATEDIFF(hour,drug_exposure_start_datetime,drug_exposure_end_datetime) FROM drug_exposure;", + targetDialect = "postgresql" + ) + expect_equal_ignore_spaces(sql, "SELECT (EXTRACT(EPOCH FROM (drug_exposure_end_datetime - drug_exposure_start_datetime)) / 3600) FROM drug_exposure;") + + sql <- translate("SELECT DATEDIFF(minute,drug_exposure_start_datetime,drug_exposure_end_datetime) FROM drug_exposure;", + targetDialect = "postgresql" + ) + expect_equal_ignore_spaces(sql, "SELECT (EXTRACT(EPOCH FROM (drug_exposure_end_datetime - drug_exposure_start_datetime)) / 60) FROM drug_exposure;") + + sql <- translate("SELECT DATEDIFF(second,drug_exposure_start_datetime,drug_exposure_end_datetime) FROM drug_exposure;", + targetDialect = "postgresql" + ) + expect_equal_ignore_spaces(sql, "SELECT EXTRACT(EPOCH FROM (drug_exposure_end_datetime - drug_exposure_start_datetime)) FROM drug_exposure;") +}) + test_that("translate sql server -> Postgres WITH SELECT", { sql <- translate("WITH cte1 AS (SELECT a FROM b) SELECT c FROM cte1;", targetDialect = "postgresql" From a7d3f2af53d8a6bb55b8eba2c5848de0af20741c Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 2 Oct 2023 14:57:45 -0400 Subject: [PATCH 6/9] Added datediff translations for Redshift --- inst/csv/replacementPatterns.csv | 16 ++++++ tests/testthat/test-translate-redshift.R | 68 ++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index 1ab848a7..4db87874 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -270,6 +270,7 @@ redshift,"O*D*R B*","ORDER BY" redshift,"ROUND(@a,@b)","ROUND(CAST(@a AS FLOAT),@b)" redshift,"ROUND(@expression,@length,@trunc)","case when @trunc = 0 then ROUND(@expression,@length) else TRUNC(@expression,@length) end" redshift,"DATEADD(dd,@days,@date)","DATEADD(day,@days,@date)" +redshift,"DATEADD(day,@days,@date)","DATEADD(day,@days,@date)" redshift,"DATEADD(m,@months,@date)","DATEADD(month,@months,@date)" redshift,"DATEADD(mm,@months,@date)","DATEADD(month,@months,@date)" redshift,"DATEADD(yyyy,@years,@date)","DATEADD(year,@years,@date)" @@ -279,12 +280,16 @@ redshift,"DATEADD(q,@n,@date)","DATEADD(quarter,@n,@date)" redshift,"DATEADD(wk,@n,@date)","DATEADD(week,@n,@date)" redshift,"DATEADD(ww,@n,@date)","DATEADD(week,@n,@date)" redshift,"DATEADD(hh,@n,@date)","DATEADD(hour,@n,@date)" +redshift,"DATEADD(hour,@n,@date)","DATEADD(hour,@n,@date)" redshift,"DATEADD(mi,@n,@date)","DATEADD(minute,@n,@date)" +redshift,"DATEADD(minute,@n,@date)","DATEADD(minute,@n,@date)" redshift,"DATEADD(n,@n,@date)","DATEADD(minute,@n,@date)" +redshift,"DATEADD(second,@n,@date)","DATEADD(second,@n,@date)" redshift,"DATEADD(ss,@n,@date)","DATEADD(second,@n,@date)" redshift,"DATEADD(mcs,@n,@date)","DATEADD(microsecond,@n,@date)" redshift,"DATEADD(@part,@n,@date)","DATEADD(@part,CAST(@n as int),@date)" redshift,"DATEDIFF(dd,@start,@end)","DATEDIFF(day,@start,@end)" +redshift,"DATEDIFF(day,@start,@end)","DATEDIFF(day,@start,@end)" redshift,"DATEDIFF(m,@start,@end)","DATEDIFF(month,@start,@end)" redshift,"DATEDIFF(mm,@start,@end)","DATEDIFF(month,@start,@end)" redshift,"DATEDIFF(yyyy,@start,@end)","DATEDIFF(year,@start,@end)" @@ -294,11 +299,15 @@ redshift,"DATEDIFF(q,@start,@end)","DATEDIFF(quarter,@start,@end)" redshift,"DATEDIFF(wk,@start,@end)","DATEDIFF(week,@start,@end)" redshift,"DATEDIFF(ww,@start,@end)","DATEDIFF(week,@start,@end)" redshift,"DATEDIFF(hh,@start,@end)","DATEDIFF(hour,@start,@end)" +redshift,"DATEDIFF(hour,@start,@end)","DATEDIFF(hour,@start,@end)" redshift,"DATEDIFF(mi,@start,@end)","DATEDIFF(minute,@start,@end)" +redshift,"DATEDIFF(minute,@start,@end)","DATEDIFF(minute,@start,@end)" redshift,"DATEDIFF(n,@start,@end)","DATEDIFF(minute,@start,@end)" redshift,"DATEDIFF(ss,@start,@end)","DATEDIFF(second,@start,@end)" +redshift,"DATEDIFF(second,@start,@end)","DATEDIFF(second,@start,@end)" redshift,"DATEDIFF(mcs,@start,@end)","DATEDIFF(microsecond,@start,@end)" redshift,"DATEDIFF_BIG(dd,@start,@end)","DATEDIFF(day,@start,@end)" +redshift,"DATEDIFF_BIG(day,@start,@end)","DATEDIFF(day,@start,@end)" redshift,"DATEDIFF_BIG(m,@start,@end)","DATEDIFF(month,@start,@end)" redshift,"DATEDIFF_BIG(mm,@start,@end)","DATEDIFF(month,@start,@end)" redshift,"DATEDIFF_BIG(yyyy,@start,@end)","DATEDIFF(year,@start,@end)" @@ -308,11 +317,15 @@ redshift,"DATEDIFF_BIG(q,@start,@end)","DATEDIFF(quarter,@start,@end)" redshift,"DATEDIFF_BIG(wk,@start,@end)","DATEDIFF(week,@start,@end)" redshift,"DATEDIFF_BIG(ww,@start,@end)","DATEDIFF(week,@start,@end)" redshift,"DATEDIFF_BIG(hh,@start,@end)","DATEDIFF(hour,@start,@end)" +redshift,"DATEDIFF_BIG(hour,@start,@end)","DATEDIFF(hour,@start,@end)" redshift,"DATEDIFF_BIG(mi,@start,@end)","DATEDIFF(minute,@start,@end)" +redshift,"DATEDIFF_BIG(minute,@start,@end)","DATEDIFF(minute,@start,@end)" redshift,"DATEDIFF_BIG(n,@start,@end)","DATEDIFF(minute,@start,@end)" redshift,"DATEDIFF_BIG(ss,@start,@end)","DATEDIFF(second,@start,@end)" +redshift,"DATEDIFF_BIG(second,@start,@end)","DATEDIFF(second,@start,@end)" redshift,"DATEDIFF_BIG(mcs,@start,@end)","DATEDIFF(microsecond,@start,@end)" redshift,"DATEPART(dd,@date)","DATEPART(day,@date)" +redshift,"DATEPART(day,@date)","DATEPART(day,@date)" redshift,"DATEPART(m,@date)","DATEPART(month,@date)" redshift,"DATEPART(mm,@date)","DATEPART(month,@date)" redshift,"DATEPART(yyyy,@date)","DATEPART(year,@date)" @@ -322,9 +335,12 @@ redshift,"DATEPART(q,@date)","DATEPART(quarter,@date)" redshift,"DATEPART(wk,@date)","DATEPART(week,@date)" redshift,"DATEPART(ww,@date)","DATEPART(week,@date)" redshift,"DATEPART(hh,@date)","DATEPART(hour,@date)" +redshift,"DATEPART(hour,@date)","DATEPART(hour,@date)" redshift,"DATEPART(mi,@date)","DATEPART(minute,@date)" +redshift,"DATEPART(minute,@date)","DATEPART(minute,@date)" redshift,"DATEPART(n,@date)","DATEPART(minute,@date)" redshift,"DATEPART(ss,@date)","DATEPART(second,@date)" +redshift,"DATEPART(second,@date)","DATEPART(second,@date)" redshift,"DATEPART(mcs,@date)","DATEPART(microsecond,@date)" redshift,"CONVERT(VARCHAR,@date,112)","TO_CHAR(@date, 'YYYYMMDD')" redshift,GETDATE(),CURRENT_DATE diff --git a/tests/testthat/test-translate-redshift.R b/tests/testthat/test-translate-redshift.R index 8d67e08b..3ec08258 100644 --- a/tests/testthat/test-translate-redshift.R +++ b/tests/testthat/test-translate-redshift.R @@ -226,6 +226,14 @@ test_that("translate sql server -> RedShift DATEADD hh", { sql, "SELECT DATEADD(hour, CAST(3 as int), drug_era_end_date) FROM drug_era;" ) + + sql <- translate("SELECT DATEADD(hour, 3, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces( + sql, + "SELECT DATEADD(hour, CAST(3 as int), drug_era_end_date) FROM drug_era;" + ) }) test_that("translate sql server -> RedShift DATEADD mi", { @@ -236,6 +244,14 @@ test_that("translate sql server -> RedShift DATEADD mi", { sql, "SELECT DATEADD(minute, CAST(3 as int), drug_era_end_date) FROM drug_era;" ) + + sql <- translate("SELECT DATEADD(minute, 3, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces( + sql, + "SELECT DATEADD(minute, CAST(3 as int), drug_era_end_date) FROM drug_era;" + ) }) test_that("translate sql server -> RedShift DATEADD ss", { @@ -246,6 +262,14 @@ test_that("translate sql server -> RedShift DATEADD ss", { sql, "SELECT DATEADD(second, CAST(3 as int), drug_era_end_date) FROM drug_era;" ) + + sql <- translate("SELECT DATEADD(second, 3, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces( + sql, + "SELECT DATEADD(second, CAST(3 as int), drug_era_end_date) FROM drug_era;" + ) }) test_that("translate sql server -> RedShift DATEADD mcs", { @@ -367,6 +391,14 @@ test_that("translate sql server -> RedShift DATEDIFF hh", { sql, "SELECT DATEDIFF(hour, drug_era_start_date, drug_era_end_date) FROM drug_era;" ) + + sql <- translate("SELECT DATEDIFF(hour, drug_era_start_date, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces( + sql, + "SELECT DATEDIFF(hour, drug_era_start_date, drug_era_end_date) FROM drug_era;" + ) }) test_that("translate sql server -> RedShift DATEDIFF mi", { @@ -377,6 +409,14 @@ test_that("translate sql server -> RedShift DATEDIFF mi", { sql, "SELECT DATEDIFF(minute, drug_era_start_date, drug_era_end_date) FROM drug_era;" ) + + sql <- translate("SELECT DATEDIFF(minute, drug_era_start_date, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces( + sql, + "SELECT DATEDIFF(minute, drug_era_start_date, drug_era_end_date) FROM drug_era;" + ) }) test_that("translate sql server -> RedShift DATEDIFF n", { @@ -397,6 +437,14 @@ test_that("translate sql server -> RedShift DATEDIFF ss", { sql, "SELECT DATEDIFF(second, drug_era_start_date, drug_era_end_date) FROM drug_era;" ) + + sql <- translate("SELECT DATEDIFF(second, drug_era_start_date, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces( + sql, + "SELECT DATEDIFF(second, drug_era_start_date, drug_era_end_date) FROM drug_era;" + ) }) test_that("translate sql server -> RedShift DATEDIFF mcs", { @@ -554,6 +602,11 @@ test_that("translate sql server -> RedShift DATEPART dd", { targetDialect = "redshift" ) expect_equal_ignore_spaces(sql, "SELECT DATEPART(day, drug_era_end_date) FROM drug_era;") + + sql <- translate("SELECT DATEPART(day, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces(sql, "SELECT DATEPART(day, drug_era_end_date) FROM drug_era;") }) test_that("translate sql server -> RedShift DATEPART m", { @@ -617,6 +670,11 @@ test_that("translate sql server -> RedShift DATEPART hh", { targetDialect = "redshift" ) expect_equal_ignore_spaces(sql, "SELECT DATEPART(hour, drug_era_end_date) FROM drug_era;") + + sql <- translate("SELECT DATEPART(hour, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces(sql, "SELECT DATEPART(hour, drug_era_end_date) FROM drug_era;") }) test_that("translate sql server -> RedShift DATEPART mi", { @@ -624,6 +682,11 @@ test_that("translate sql server -> RedShift DATEPART mi", { targetDialect = "redshift" ) expect_equal_ignore_spaces(sql, "SELECT DATEPART(minute, drug_era_end_date) FROM drug_era;") + + sql <- translate("SELECT DATEPART(minute, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces(sql, "SELECT DATEPART(minute, drug_era_end_date) FROM drug_era;") }) test_that("translate sql server -> RedShift DATEPART n", { @@ -638,6 +701,11 @@ test_that("translate sql server -> RedShift DATEPART ss", { targetDialect = "redshift" ) expect_equal_ignore_spaces(sql, "SELECT DATEPART(second, drug_era_end_date) FROM drug_era;") + + sql <- translate("SELECT DATEPART(second, drug_era_end_date) FROM drug_era;", + targetDialect = "redshift" + ) + expect_equal_ignore_spaces(sql, "SELECT DATEPART(second, drug_era_end_date) FROM drug_era;") }) test_that("translate sql server -> RedShift DATEPART mcs", { From 31cb0c7a120d29f166767561c353b507f4cec1e1 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Sun, 11 Feb 2024 12:39:37 -0500 Subject: [PATCH 7/9] Removed date translations that make no changes --- inst/csv/replacementPatterns.csv | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index 489d99a6..4e23f80b 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -277,7 +277,6 @@ redshift,"O*D*R B*","ORDER BY" redshift,"ROUND(@a,@b)","ROUND(CAST(@a AS FLOAT),@b)" redshift,"ROUND(@expression,@length,@trunc)","case when @trunc = 0 then ROUND(@expression,@length) else TRUNC(@expression,@length) end" redshift,"DATEADD(dd,@days,@date)","DATEADD(day,@days,@date)" -redshift,"DATEADD(day,@days,@date)","DATEADD(day,@days,@date)" redshift,"DATEADD(m,@months,@date)","DATEADD(month,@months,@date)" redshift,"DATEADD(mm,@months,@date)","DATEADD(month,@months,@date)" redshift,"DATEADD(yyyy,@years,@date)","DATEADD(year,@years,@date)" @@ -287,16 +286,12 @@ redshift,"DATEADD(q,@n,@date)","DATEADD(quarter,@n,@date)" redshift,"DATEADD(wk,@n,@date)","DATEADD(week,@n,@date)" redshift,"DATEADD(ww,@n,@date)","DATEADD(week,@n,@date)" redshift,"DATEADD(hh,@n,@date)","DATEADD(hour,@n,@date)" -redshift,"DATEADD(hour,@n,@date)","DATEADD(hour,@n,@date)" redshift,"DATEADD(mi,@n,@date)","DATEADD(minute,@n,@date)" -redshift,"DATEADD(minute,@n,@date)","DATEADD(minute,@n,@date)" redshift,"DATEADD(n,@n,@date)","DATEADD(minute,@n,@date)" -redshift,"DATEADD(second,@n,@date)","DATEADD(second,@n,@date)" redshift,"DATEADD(ss,@n,@date)","DATEADD(second,@n,@date)" redshift,"DATEADD(mcs,@n,@date)","DATEADD(microsecond,@n,@date)" redshift,"DATEADD(@part,@n,@date)","DATEADD(@part,CAST(@n as int),@date)" redshift,"DATEDIFF(dd,@start,@end)","DATEDIFF(day,@start,@end)" -redshift,"DATEDIFF(day,@start,@end)","DATEDIFF(day,@start,@end)" redshift,"DATEDIFF(m,@start,@end)","DATEDIFF(month,@start,@end)" redshift,"DATEDIFF(mm,@start,@end)","DATEDIFF(month,@start,@end)" redshift,"DATEDIFF(yyyy,@start,@end)","DATEDIFF(year,@start,@end)" @@ -306,12 +301,9 @@ redshift,"DATEDIFF(q,@start,@end)","DATEDIFF(quarter,@start,@end)" redshift,"DATEDIFF(wk,@start,@end)","DATEDIFF(week,@start,@end)" redshift,"DATEDIFF(ww,@start,@end)","DATEDIFF(week,@start,@end)" redshift,"DATEDIFF(hh,@start,@end)","DATEDIFF(hour,@start,@end)" -redshift,"DATEDIFF(hour,@start,@end)","DATEDIFF(hour,@start,@end)" redshift,"DATEDIFF(mi,@start,@end)","DATEDIFF(minute,@start,@end)" -redshift,"DATEDIFF(minute,@start,@end)","DATEDIFF(minute,@start,@end)" redshift,"DATEDIFF(n,@start,@end)","DATEDIFF(minute,@start,@end)" redshift,"DATEDIFF(ss,@start,@end)","DATEDIFF(second,@start,@end)" -redshift,"DATEDIFF(second,@start,@end)","DATEDIFF(second,@start,@end)" redshift,"DATEDIFF(mcs,@start,@end)","DATEDIFF(microsecond,@start,@end)" redshift,"DATEDIFF_BIG(dd,@start,@end)","DATEDIFF(day,@start,@end)" redshift,"DATEDIFF_BIG(day,@start,@end)","DATEDIFF(day,@start,@end)" @@ -332,7 +324,6 @@ redshift,"DATEDIFF_BIG(ss,@start,@end)","DATEDIFF(second,@start,@end)" redshift,"DATEDIFF_BIG(second,@start,@end)","DATEDIFF(second,@start,@end)" redshift,"DATEDIFF_BIG(mcs,@start,@end)","DATEDIFF(microsecond,@start,@end)" redshift,"DATEPART(dd,@date)","DATEPART(day,@date)" -redshift,"DATEPART(day,@date)","DATEPART(day,@date)" redshift,"DATEPART(m,@date)","DATEPART(month,@date)" redshift,"DATEPART(mm,@date)","DATEPART(month,@date)" redshift,"DATEPART(yyyy,@date)","DATEPART(year,@date)" @@ -342,12 +333,9 @@ redshift,"DATEPART(q,@date)","DATEPART(quarter,@date)" redshift,"DATEPART(wk,@date)","DATEPART(week,@date)" redshift,"DATEPART(ww,@date)","DATEPART(week,@date)" redshift,"DATEPART(hh,@date)","DATEPART(hour,@date)" -redshift,"DATEPART(hour,@date)","DATEPART(hour,@date)" redshift,"DATEPART(mi,@date)","DATEPART(minute,@date)" -redshift,"DATEPART(minute,@date)","DATEPART(minute,@date)" redshift,"DATEPART(n,@date)","DATEPART(minute,@date)" redshift,"DATEPART(ss,@date)","DATEPART(second,@date)" -redshift,"DATEPART(second,@date)","DATEPART(second,@date)" redshift,"DATEPART(mcs,@date)","DATEPART(microsecond,@date)" redshift,"CONVERT(VARCHAR,@date,112)","TO_CHAR(@date, 'YYYYMMDD')" redshift,GETDATE(),CURRENT_DATE From f068c698442410f789e3f523d39147c35747457d Mon Sep 17 00:00:00 2001 From: Thomas White Date: Sun, 11 Feb 2024 12:56:39 -0500 Subject: [PATCH 8/9] Added negative interval translations for Spark --- inst/csv/replacementPatterns.csv | 3 +++ tests/testthat/test-translate-spark.R | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index 4e23f80b..e6466d86 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -1080,6 +1080,9 @@ spark,"HASHBYTES('MD5',@a)","MD5(@a)" spark,"CONVERT(VARBINARY, CONCAT('0x', @a), 1)","CAST(CONCAT('x', @a) AS BIT(32))" spark,"CONVERT(DATE, @a)","TO_DATE(@a, 'yyyy-MM-dd')" spark,"DATEPART(@part, @date)","DATE_PART('@part', @date)" +spark,"DATEADD(second, -1 * @seconds,@datetime)",(@datetime - INTERVAL @seconds second) +spark,"DATEADD(minute, -1 * @minutes,@datetime)",(@datetime - INTERVAL @minutes minute) +spark,"DATEADD(hour, -1 * @hours,@datetime)",(@datetime - INTERVAL @hours hour) spark,"DATEADD(second,@seconds,@datetime)",(@datetime + INTERVAL @seconds second) spark,"DATEADD(minute,@minutes,@datetime)",(@datetime + INTERVAL @minutes minute) spark,"DATEADD(hour,@hours,@datetime)",(@datetime + INTERVAL @hours hour) diff --git a/tests/testthat/test-translate-spark.R b/tests/testthat/test-translate-spark.R index 517ea43e..657160fd 100644 --- a/tests/testthat/test-translate-spark.R +++ b/tests/testthat/test-translate-spark.R @@ -55,6 +55,23 @@ test_that("translate sql server -> spark convert date", { test_that("translate sql server -> spark dateadd", { + # Need custom translation pattern for negative intervals in Spark + sql <- translate("SELECT dateadd(second, -1 * 2, '2019-01-01 00:00:00')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT ('2019-01-01 00:00:00' - INTERVAL 2 second)") + + sql <- translate("SELECT dateadd(minute, -1 * 3, '2019-01-01 00:00:00')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT ('2019-01-01 00:00:00' - INTERVAL 3 minute)") + + sql <- translate("SELECT dateadd(hour, -1 * 4, '2019-01-01 00:00:00')", + targetDialect = "spark" + ) + expect_equal_ignore_spaces(sql, "SELECT ('2019-01-01 00:00:00' - INTERVAL 4 hour)") + + # Positive intervals have typical translation patterns sql <- translate("SELECT dateadd(second, 1, '2019-01-01 00:00:00')", targetDialect = "spark" ) From 3bfb529f3ec4652bf576502bfb0de7d2ac3fdecf Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 26 Feb 2024 15:06:34 -0500 Subject: [PATCH 9/9] Updated spark translations - has supported DATE_ADD() since version 8.0 - e.g. for nearly 2 years --- inst/csv/replacementPatterns.csv | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/inst/csv/replacementPatterns.csv b/inst/csv/replacementPatterns.csv index e6466d86..cd5b1925 100644 --- a/inst/csv/replacementPatterns.csv +++ b/inst/csv/replacementPatterns.csv @@ -1080,21 +1080,18 @@ spark,"HASHBYTES('MD5',@a)","MD5(@a)" spark,"CONVERT(VARBINARY, CONCAT('0x', @a), 1)","CAST(CONCAT('x', @a) AS BIT(32))" spark,"CONVERT(DATE, @a)","TO_DATE(@a, 'yyyy-MM-dd')" spark,"DATEPART(@part, @date)","DATE_PART('@part', @date)" -spark,"DATEADD(second, -1 * @seconds,@datetime)",(@datetime - INTERVAL @seconds second) -spark,"DATEADD(minute, -1 * @minutes,@datetime)",(@datetime - INTERVAL @minutes minute) -spark,"DATEADD(hour, -1 * @hours,@datetime)",(@datetime - INTERVAL @hours hour) -spark,"DATEADD(second,@seconds,@datetime)",(@datetime + INTERVAL @seconds second) -spark,"DATEADD(minute,@minutes,@datetime)",(@datetime + INTERVAL @minutes minute) -spark,"DATEADD(hour,@hours,@datetime)",(@datetime + INTERVAL @hours hour) -spark,"DATEADD(d,@days,@date)","date_add(@date, @days)" -spark,"DATEADD(dd,@days,@date)","date_add(@date, @days)" -spark,"DATEADD(day,@days,@date)","date_add(@date, @days)" -spark,"DATEADD(m,@months,@date)",(@date + INTERVAL @months month) -spark,"DATEADD(mm,@months,@date)",(@date + INTERVAL @months month) -spark,"DATEADD(month,@months,@date)",(@date + INTERVAL @months month) -spark,"DATEADD(yy,@years,@date)",(@date + INTERVAL @years year) -spark,"DATEADD(yyyy,@years,@date)",(@date + INTERVAL @years year) -spark,"DATEADD(year,@years,@date)",(@date + INTERVAL @years year) +spark,"DATEADD(second,@seconds,@datetime)",DATE_ADD(second,@seconds,@datetime) +spark,"DATEADD(minute,@minutes,@datetime)",DATE_ADD(minute,@minutes,@datetime) +spark,"DATEADD(hour,@hours,@datetime)",DATE_ADD(hour,@hours,@datetime) +spark,"DATEADD(d,@days,@date)",DATE_ADD(day,@days,@date) +spark,"DATEADD(dd,@days,@date)",DATE_ADD(day,@days,@date) +spark,"DATEADD(day,@days,@date)",DATE_ADD(day,@days,@date) +spark,"DATEADD(m,@months,@date)",DATE_ADD(month,@months,@date) +spark,"DATEADD(mm,@months,@date)",DATE_ADD(month,@months,@date) +spark,"DATEADD(month,@months,@date)",DATE_ADD(month,@months,@date) +spark,"DATEADD(yy,@years,@date)",DATE_ADD(year,@years,@date) +spark,"DATEADD(yyyy,@years,@date)",DATE_ADD(year,@years,@date) +spark,"DATEADD(year,@years,@date)",DATE_ADD(year,@years,@date) spark,"date_add(@date, @(-?[0-9]+)a.0)","date_add(@date, @a)" spark,INTERVAL @(-?[0-9]+)a.0,INTERVAL @a spark,"DATEDIFF(second,@start,@end)","datediff(second,@end,@start)"