Updates

NAMESPACE

@@ -129,6 +129,7 @@ importFrom(sparklyr,spark_write_text)
 importFrom(tidyr,pivot_longer)
 importFrom(tidyselect,matches)
 importFrom(tidyselect,tidyselect_data_has_predicates)
+importFrom(utils,compareVersion)
 importFrom(utils,head)
 importFrom(utils,type.convert)
 importFrom(vctrs,vec_as_names)

R/install-pyspark.R

@@ -68,7 +68,7 @@ install_pyspark <- function(envname = "r-sparklyr",
     method = method,
     python_version = python_version,
     pip = TRUE,
-    ...
+    ... = ...
   )
 }
 

R/local-connect.R

@@ -1,17 +1,24 @@
 #' Starts Spark Connect locally
 #' @param version Spark version to use (3.4 or above)
 #' @param scala_version Acceptable Scala version of packages to be loaded
+#' @param include_args Flag that indicates wether to add the additional arguments
+#' to the command that starts the service. At this time, only the 'packages'
+#' argument is submitted.
 #' @param ... Optional arguments; currently unused
 #' @export
 spark_connect_service_start <- function(version = "3.4",
                                         scala_version = "2.12",
+                                        include_args = TRUE,
                                         ...) {
   get_version <- spark_install_find(version = version)
   cmd <- path(get_version$sparkVersionDir, "sbin", "start-connect-server.sh")
   args <- c(
     "--packages",
     glue("org.apache.spark:spark-connect_{scala_version}:{get_version$sparkVersion}")
   )
+  if(!include_args) {
+    args <- ""
+  }
   prs <- process$new(
     command = cmd,
     args = args,

R/package.R

@@ -17,7 +17,7 @@
 #' @importFrom rlang is_string is_character as_utf8_character parse_exprs
 #' @importFrom methods new is setOldClass
 #' @importFrom tidyselect matches
-#' @importFrom utils head type.convert
+#' @importFrom utils head type.convert compareVersion
 #' @importFrom tidyr pivot_longer
 #' @importFrom vctrs vec_as_names
 #' @importFrom processx process