Merge pull request #102 from mlverse/updates

Improvements to deploy, adds `requirements_write()`

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: pysparklyr
 Title: Provides a 'PySpark' Back-End for the 'sparklyr' Package
-Version: 0.1.2.9006
+Version: 0.1.2.9007
 Authors@R: c(
     person("Edgar", "Ruiz", , "[email protected]", role = c("aut", "cre")),
     person(given = "Posit Software, PBC", role = c("cph", "fnd"))

NAMESPACE

@@ -191,5 +191,6 @@ importFrom(utils,capture.output)
 importFrom(utils,compareVersion)
 importFrom(utils,head)
 importFrom(utils,installed.packages)
+importFrom(utils,menu)
 importFrom(utils,type.convert)
 importFrom(vctrs,vec_as_names)

NEWS.md

@@ -9,6 +9,10 @@
 * Adds `deploy_databricks()` function. It will simplify publishing to Posit
 Connect by automating much of the needed setup, and triggers the publication.
 
+* Adds `requirements_write()` function. It will inventory the Python libraries
+loaded in a given Python environment and create the 'requirements.txt'. This
+is in an effort to make it easier to republish deployed content.
+
 ### Improvements
 
 * Improvements to the RStudio connections snippet. It now adapts for when the

R/deploy.R

@@ -74,7 +74,7 @@ deploy_databricks <- function(
     }
   }
   if (!is.null(host)) {
-    env_var_message <- c(" " = glue("|- Host: {host}"))
+    env_var_message <- c("i" = paste0("{.header Host URL:} ", host))
   } else {
     var_error <- c(" " = paste0(
       "{.header - No host URL was provided or found. Please either set the}",
@@ -98,7 +98,7 @@ deploy_databricks <- function(
   if (!is.null(token)) {
     env_var_message <- c(
       env_var_message,
-      " " = glue("|- Token: '<REDACTED>'")
+      " " = "{.header Token:} '<REDACTED>'"
     )
   } else {
     var_error <- c(var_error, " " = paste0(
@@ -154,16 +154,14 @@ deploy <- function(
       server <- rs_accounts$server[1]
     }
     if (nrow(rs_accounts > 1)) {
-      accts_msg <- "Change Publishing Target (Posit Connect server)"
+      accts_msg <- "Change 'Posit server'"
     }
   }
   cli_div(theme = cli_colors())
   cli_h1("Starting deployment")
-  check_rstudio <- try(RStudio.Version(), silent = TRUE)
-  in_rstudio <- !inherits(check_rstudio, "try-error")
   editor_doc <- NULL
   if (is.null(appDir)) {
-    if (interactive() && in_rstudio) {
+    if (check_interactive() && check_rstudio()) {
       editor_doc <- getSourceEditorContext()
       if (!is.null(editor_doc)) {
         appDir <- dirname(editor_doc$path)
@@ -175,46 +173,49 @@ deploy <- function(
   }
   appDir <- path(appDir)
 
-  cli_inform("{.class - App and Spark -}")
-  cli_alert_info("{.header Source: {.emph '{appDir}'}}")
+  cli_alert_info("{.header Source directory: {.emph {appDir}}}")
   python <- deploy_find_environment(
     python = python,
     version = version,
     backend = backend
   )
-  cli_inform("{.class - Publishing target -}")
-  cli_alert_info("{.header Server:} {server} | {.header Account:} {account}")
+  cli_inform(c(
+    "i" = "{.header Posit server:} {.emph {server}}",
+    " " = "{.header Account name:} {.emph {account}}"
+  ))
   if (!is.null(env_var_message)) {
-    cli_bullets(c("i" = "{.header Environment variables:}", env_var_message))
+    cli_bullets(env_var_message)
   }
   cli_inform("")
-  if (interactive() && confirm) {
-    cli_inform("Does everything look correct?")
-    choice <- utils::menu(choices = c("Yes", "No", accts_msg))
+  if (check_interactive() && confirm) {
+    cli_inform("{.header Does everything look correct?}")
+    choice <- menu(choices = c("Yes", "No", accts_msg))
     if (choice == 2) {
       return(invisible())
     }
     if (choice == 3) {
       chr_accounts <- rs_accounts %>%
         transpose() %>%
         map_chr(~ glue("Server: {.x$server} | Account: {.x$name}"))
-      choice <- utils::menu(title = "Select publishing target:", chr_accounts)
+      choice <- menu(title = "Select publishing target:", chr_accounts)
+      server <- rs_accounts$server[choice]
+      account <- rs_accounts$name[choice]
     }
   }
 
   req_file <- path(appDir, "requirements.txt")
   prev_deployments <- deployments(appDir)
-  if(!file_exists(req_file) && nrow(prev_deployments) == 0 && interactive()) {
+  if(!file_exists(req_file) && nrow(prev_deployments) == 0 && check_interactive()) {
     cli_inform(c(
       "{.header Would you like to create the 'requirements.txt' file?}",
       "{.class Why consider? This will allow you to skip using `version` or `cluster_id`}"
     ))
-    choice <- utils::menu(choices = c("Yes", "No"))
+    choice <- menu(choices = c("Yes", "No"))
     if(choice == 1) {
       requirements_write(
         destfile = req_file,
         python = python
-        )
+      )
     }
   }
 
@@ -236,11 +237,7 @@ deploy_find_environment <- function(
   ret <- NULL
   failed <- NULL
   env_name <- ""
-  cli_progress_step(
-    msg = "Searching and validating Python path",
-    msg_done = "{.header Python:{.emph '{ret}'}}",
-    msg_failed = "Environment not found: {.emph {failed}}"
-  )
+  exe_py <- py_exe()
   if (is.null(python)) {
     if (!is.null(version)) {
       env_name <- use_envname(
@@ -255,9 +252,8 @@ deploy_find_environment <- function(
       }
       if (is.null(ret)) failed <- env_name
     } else {
-      py_exe_path <- py_exe()
-      if (grepl("r-sparklyr-", py_exe_path)) {
-        ret <- py_exe_path
+      if (grepl("r-sparklyr-", exe_py)) {
+        ret <- exe_py
       } else {
         failed <- "Please pass a 'version' or a 'cluster_id'"
       }
@@ -271,15 +267,16 @@ deploy_find_environment <- function(
     }
   }
   if (is.null(ret)) {
-    exe_py <- py_exe()
-    if(exe_py == "") {
-      cli_progress_done(result = "failed")
+
+    if(is.null(exe_py)) {
       cli_abort("No Python environment could be found")
     } else {
       ret <- exe_py
     }
   } else {
     ret <- path_expand(ret)
   }
+  cli_bullets(c("i" = "{.header Python:} {ret}"))
   ret
 }
+

R/ml-utils.R

@@ -136,14 +136,14 @@ ml_installed <- function() {
   if (!all(find_ml)) {
     cli_div(theme = cli_colors())
     msg1 <- "Required Python libraries to run ML functions are missing"
-    if (interactive()) {
+    if (check_interactive()) {
       missing_ml <- ml_libraries[!find_ml]
       cli_alert_warning(msg1)
       cli_bullets(c(
         " " = "{.header Could not find: {missing_ml}}",
         " " = "Do you wish to install? {.class (This will be a one time operation)}"
       ))
-      choice <- utils::menu(choices = c("Yes", "Cancel"))
+      choice <- menu(choices = c("Yes", "Cancel"))
       if (choice == 1) {
         py_install(missing_ml)
       }

R/package.R

@@ -30,7 +30,7 @@
 #' @importFrom processx process
 #' @importFrom rstudioapi jobRunScript showQuestion getSourceEditorContext
 #' @importFrom stats terms
-#' @importFrom utils capture.output installed.packages
+#' @importFrom utils capture.output installed.packages menu
 #' @importFrom magrittr %>%
 #' @import reticulate
 #' @import dbplyr

R/python-install.R

@@ -440,6 +440,8 @@ check_rstudio <- function() {
   }
 }
 
+check_interactive <- function() interactive()
+
 build_job_code <- function(args) {
   args$as_job <- NULL
   args$method <- args$method[[1]]

R/python-requirements.R

@@ -1,4 +1,7 @@
-#' Write the 'requirements.txt' file
+#' Writes the 'requirements.txt' file, containing the needed Python libraries
+#' @description This is a helper function that it is meant to be used for deployments
+#' of the document or application. By default, `deploy_databricks()` will run this
+#' function the first time you use that function to deploy content to Posit Connect.
 #' @param envname The name of, or path to, a Python virtual environment.
 #' @param destfile Target path for the requirements file. Defaults to 'requirements.txt'.
 #' @param overwrite Replace the contents of the file if it already exists?

R/python-use-envname.R

@@ -95,7 +95,7 @@ use_envname <- function(
         " " = msg_2,
         " " = "Do you wish to install {con_label} version {version}?"
       ))
-      choice <- utils::menu(choices = c("Yes", "No", "Cancel"))
+      choice <- menu(choices = c("Yes", "No", "Cancel"))
       if (choice == 1) {
         ret <- set_names(envname, "prompt")
         rlang::exec(

tests/testthat/helper-init.R

@@ -3,6 +3,7 @@
 .test_env$lr_model <- NULL
 .test_env$env <- NULL
 .test_env$started <- NULL
+.test_env$dbr <- NULL
 
 use_test_env <- function() {
   if (is.null(.test_env$env)) {

tests/testthat/helper-utils.R

@@ -96,3 +96,19 @@ tests_enable_all <- function() {
     }
   )
 }
+
+test_databricks_cluster_id <- function() {
+  Sys.getenv("DATABRICKS_CLUSTER_ID")
+}
+
+test_databricks_cluster_version <- function() {
+  if(is.null(.test_env$dbr)) {
+    dbr <- databricks_dbr_version(
+      cluster_id = test_databricks_cluster_id(),
+      host = databricks_host(),
+      token = databricks_token()
+    )
+    .test_env$dbr <- dbr
+  }
+  .test_env$dbr
+}