Accounts for new Spark dataframe header output, skips spark_apply() w…

…ith barrier if < Spark 3.5
mlverse · Feb 2, 2024 · a300510 · a300510
1 parent 48c4715
commit a300510
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 4 deletions.
diff --git a/tests/testthat/_snaps/pivot-longer.md b/tests/testthat/_snaps/pivot-longer.md
@@ -1,10 +1,10 @@
 # Pivot longer
 
     Code
-      tbl_pivot %>% tidyr::pivot_longer(-id, names_to = c(".value", "n"), names_sep = "_")
+      tbl_pivot %>% tidyr::pivot_longer(-id, names_to = c(".value", "n"), names_sep = "_") %>%
+        collect()
     Output
-      # Source:   table<sparklyr_tmp_table_60b5433c_d4da_42a5_86cd_2a2fffdc57b2> [4 x 5]
-      # Database: spark_connection
+      # A tibble: 4 x 5
         id    n         z     y     x
         <chr> <chr> <dbl> <dbl> <dbl>
       1 A     1         1     2     3

diff --git a/tests/testthat/test-pivot-longer.R b/tests/testthat/test-pivot-longer.R
@@ -12,7 +12,8 @@ test_that("Pivot longer", {
       tidyr::pivot_longer(
         -id,
         names_to = c(".value", "n"), names_sep = "_"
-      )
+      ) %>%
+      collect()
   )
 })
 

diff --git a/tests/testthat/test-sparklyr-spark-apply.R b/tests/testthat/test-sparklyr-spark-apply.R
@@ -4,6 +4,7 @@ test_that("spark_apply() works", {
     spark_apply(tbl_mtcars, nrow, group_by = "am", columns = "am double, x long"),
     "tbl_spark"
   )
+  skip_spark_min_version(3.5)
   expect_s3_class(
     spark_apply(tbl_mtcars, function(x) x),
     "tbl_spark"