diff --git a/NAMESPACE b/NAMESPACE index dbf0bff..191ff4c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -129,6 +129,7 @@ importFrom(sparklyr,spark_write_text) importFrom(tidyr,pivot_longer) importFrom(tidyselect,matches) importFrom(tidyselect,tidyselect_data_has_predicates) +importFrom(utils,compareVersion) importFrom(utils,head) importFrom(utils,type.convert) importFrom(vctrs,vec_as_names) diff --git a/R/install-pyspark.R b/R/install-pyspark.R index 2b92dea..9e606bb 100644 --- a/R/install-pyspark.R +++ b/R/install-pyspark.R @@ -68,7 +68,7 @@ install_pyspark <- function(envname = "r-sparklyr", method = method, python_version = python_version, pip = TRUE, - ... + ... = ... ) } diff --git a/R/local-connect.R b/R/local-connect.R index 1080fc1..d5f71de 100644 --- a/R/local-connect.R +++ b/R/local-connect.R @@ -1,10 +1,14 @@ #' Starts Spark Connect locally #' @param version Spark version to use (3.4 or above) #' @param scala_version Acceptable Scala version of packages to be loaded +#' @param include_args Flag that indicates wether to add the additional arguments +#' to the command that starts the service. At this time, only the 'packages' +#' argument is submitted. #' @param ... Optional arguments; currently unused #' @export spark_connect_service_start <- function(version = "3.4", scala_version = "2.12", + include_args = TRUE, ...) { get_version <- spark_install_find(version = version) cmd <- path(get_version$sparkVersionDir, "sbin", "start-connect-server.sh") @@ -12,6 +16,9 @@ spark_connect_service_start <- function(version = "3.4", "--packages", glue("org.apache.spark:spark-connect_{scala_version}:{get_version$sparkVersion}") ) + if(!include_args) { + args <- "" + } prs <- process$new( command = cmd, args = args, diff --git a/R/package.R b/R/package.R index 27e0389..eea8d24 100644 --- a/R/package.R +++ b/R/package.R @@ -17,7 +17,7 @@ #' @importFrom rlang is_string is_character as_utf8_character parse_exprs #' @importFrom methods new is setOldClass #' @importFrom tidyselect matches -#' @importFrom utils head type.convert +#' @importFrom utils head type.convert compareVersion #' @importFrom tidyr pivot_longer #' @importFrom vctrs vec_as_names #' @importFrom processx process diff --git a/man/spark_connect_service_start.Rd b/man/spark_connect_service_start.Rd index 4bfbfeb..bcf4fb2 100644 --- a/man/spark_connect_service_start.Rd +++ b/man/spark_connect_service_start.Rd @@ -5,7 +5,12 @@ \alias{spark_connect_service_stop} \title{Starts Spark Connect locally} \usage{ -spark_connect_service_start(version = "3.4", scala_version = "2.12", ...) +spark_connect_service_start( + version = "3.4", + scala_version = "2.12", + include_args = TRUE, + ... +) spark_connect_service_stop(version = "3.4", ...) } @@ -14,6 +19,10 @@ spark_connect_service_stop(version = "3.4", ...) \item{scala_version}{Acceptable Scala version of packages to be loaded} +\item{include_args}{Flag that indicates wether to add the additional arguments +to the command that starts the service. At this time, only the 'packages' +argument is submitted.} + \item{...}{Optional arguments; currently unused} } \description{