Merge branch 'dev'

* dev:
  Rebuild documentation
  Minor refactoring
  Move helper functions to testthat
  Rename file
  Use message() instead of writeLines()
  Use markdown in roxygen #57
  Add help for `marker`
  Ignore Visual Studio RTVS artefacts
  Ignore .Rproj
  Use markdown in roxygen #57

.Rbuildignore

@@ -1,4 +1,7 @@
 ^.*\.Rproj$
 ^\.Rproj\.user$
+.rxproj$
+.rxproj.user$
+.vs/*
 CONTRIBUTING.md
 README.md

.gitignore

@@ -1,4 +1,4 @@
-.Rproj.user
+*.Rproj.user
 .Rhistory
 .RData
 *.Rproj

DESCRIPTION

@@ -1,35 +1,36 @@
 Package: AzureSMR
 Title: Package for Interacting with Azure Services Manager
-Description: Helps users to manage Azure Services and objects from within an R Session.
-    This includes Azure Storage (e.g. containers and blobs), Virtual Machines and HDInsight (Spark, Hive).
-    Requires that an Active Directory application and service principal has been configured 
-    to access Azure resources.
+Description: Helps users to manage Azure Services and objects from within an
+    R Session. This includes Azure Storage (e.g. containers and blobs), Virtual
+    Machines and HDInsight (Spark, Hive). Requires that an Active Directory
+    application and service principal has been configured to access Azure resources.
 Type: Package
 Version: 0.2.2
 Date: 2016-12-22
 Authors@R: c(
       person(family="Microsoft Corporation", role="cph"),
-      person("Alan", "Weaver", role=c("aut"), email="[email protected]"),
-      person("Andrie", "de Vries", role=c("aut", "cre"), email="[email protected]")
+      person("Alan", "Weaver", role=c("aut", "cre"), email="[email protected]"),
+      person("Andrie", "de Vries", role=c("aut"), email="[email protected]")
     )
 Copyright: Microsoft
 License: MIT + file LICENSE
 URL: https://github.com/Microsoft/AzureSMR
 BugReports: https://github.com/Microsoft/AzureSMR/issues
 NeedsCompilation: no
 Imports:
-    httr, 
-    jsonlite, 
+    httr,
+    jsonlite,
     XML,
-    plyr, 
-    base64enc, 
+    plyr,
+    base64enc,
     digest
 Depends:
     R(>= 3.0.0)
-Suggests: 
-    knitr, 
+Suggests:
+    knitr,
     rmarkdown,
     testthat
 VignetteBuilder: knitr
 LazyData: TRUE
-RoxygenNote: 6.0.0
+RoxygenNote: 6.0.1
+Roxygen: list(markdown = TRUE)

NAMESPACE

@@ -76,6 +76,7 @@ importFrom(httr,http_status)
 importFrom(httr,status_code)
 importFrom(jsonlite,fromJSON)
 importFrom(plyr,rbind.fill)
+importFrom(utils,URLencode)
 importFrom(utils,browseURL)
 importFrom(utils,ls.str)
 importFrom(utils,str)

R/AzureAuthenticate.R

@@ -3,42 +3,44 @@
 #' @inheritParams setAzureContext
 #' @param verbose Print Tracing information (Default False)
 #'
-#' @note See \url{https://azure.microsoft.com/en-us/documentation/articles/resource-group-create-service-principal-portal/} to learn how to set up an Active directory application
+#' @note See \url{https://azure.microsoft.com/en-us/documentation/articles/resource-group-create-service-principal-portal/} for instructions to set up an Active Directory application
 #' @references \url{https://azure.microsoft.com/en-us/documentation/articles/resource-group-create-service-principal-portal/}
 #'
 #' @return Retunrs Azure Tokem and sets AzureContext Token
 #' @family Resources
+#' 
+#' @importFrom utils URLencode
 #' @export
 azureAuthenticate <- function(azureActiveContext, tenantID, clientID, authKey, verbose = FALSE) {
 
   if (missing(tenantID)) {
-    AtenantID <- azureActiveContext$tenantID
-  } else (AtenantID <- tenantID)
+    tenantID <- azureActiveContext$tenantID
+  } else (tenantID <- tenantID)
   if (missing(clientID)) {
-    AclientID <- azureActiveContext$clientID
-  } else (AclientID <- clientID)
+    clientID <- azureActiveContext$clientID
+  } else (clientID <- clientID)
   if (missing(authKey)) {
-    AauthKey <- azureActiveContext$authKey
-  } else (AauthKey <- authKey)
+    authKey <- azureActiveContext$authKey
+  } else (authKey <- authKey)
 
-  if (!length(AtenantID)) {
+  if (!length(tenantID)) {
     stop("Error: No tenantID provided: Use tenantID argument or set in AzureContext")
   }
-  if (!length(AclientID)) {
+  if (!length(clientID)) {
     stop("Error: No clientID provided: Use clientID argument or set in AzureContext")
   }
-  if (!length(AauthKey)) {
+  if (!length(authKey)) {
     stop("Error: No authKey provided: Use authKey argument or set in AzureContext")
   }
   verbosity <- if (verbose)
     httr::verbose(TRUE) else NULL
 
-  URLGT <- paste0("https://login.microsoftonline.com/", AtenantID, "/oauth2/token?api-version=1.0")
+  URLGT <- paste0("https://login.microsoftonline.com/", tenantID, "/oauth2/token?api-version=1.0")
 
-  AauthKeyE <- URLencode(AauthKey, reserved = TRUE)
+  authKeyEncoded <- URLencode(authKey, reserved = TRUE)
 
   bodyGT <- paste0("grant_type=client_credentials&resource=https%3A%2F%2Fmanagement.azure.com%2F&client_id=",
-                   AclientID, "&client_secret=", AauthKeyE)
+                   clientID, "&client_secret=", authKeyEncoded)
 
   r <- httr::POST(URLGT,
                   add_headers(
@@ -53,9 +55,9 @@ azureAuthenticate <- function(azureActiveContext, tenantID, clientID, authKey, v
 
 
   azureActiveContext$Token  <- AT
-  azureActiveContext$tenantID    <- AtenantID
-  azureActiveContext$clientID    <- AclientID
-  azureActiveContext$authKey    <- AauthKey
+  azureActiveContext$tenantID    <- tenantID
+  azureActiveContext$clientID    <- clientID
+  azureActiveContext$authKey    <- authKey
   azureActiveContext$EXPIRY <- Sys.time() + 3598
   SUBS <- azureListSubscriptions(azureActiveContext)
   return("Authentication Suceeded : Key Obtained")

R/AzureBlob.R

@@ -5,10 +5,10 @@
 #' @inheritParams azureAuthenticate
 #' @inheritParams azureSAGetKey
 #'
-#' @param maxresults Optional. Specifies the maximum number of blobs to return, including all BlobPrefix elements. If the request does not specify maxresults or specifies a value greater than 5,000, the server will return up to 5,000 items.  Setting maxresults to a value less than or equal to zero results in error response code 400(Bad Request) .
+#' @param maxresults Optional. Specifies the maximum number of blobs to return, including all BlobPrefix elements. If the request does not specify maxresults or specifies a value greater than 5,000, the server will return up to 5,000 items.  Setting maxresults to a value less than or equal to zero results in error response code 400 (Bad Request).
 #' @param prefix Optional. Filters the results to return only blobs whose names begin with the specified prefix.
 #' @param delimiter Optional. When the request includes this parameter, the operation returns a BlobPrefix element in the response body that acts as a placeholder for all blobs whose names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string.
-#' @param Optional. A string value that identifies the portion of the list to be returned with the next list operation. The operation returns a marker value within the response body if the list returned was not complete. The marker value may then be used in a subsequent call to request the next set of list items.  The marker value is opaque to the client.
+#' @param marker Optional. A string value that identifies the portion of the list to be returned with the next list operation. The operation returns a marker value within the response body if the list returned was not complete. The marker value may then be used in a subsequent call to request the next set of list items.  The marker value is opaque to the client.
 #'
 #' @return Returns a data frame. This data frame has an attribute called "marker" that can be used with the "marker" argument to return the next set of values.
 #' @family blob store functions
@@ -275,7 +275,7 @@ azureBlobLS <- function(azureActiveContext, directory, recursive = FALSE,
 #' @inheritParams azureSAGetKey
 #' @inheritParams azureBlobLS
 
-#' @param type String, either "text" or "raw". Passed to \code{\link[httr]{content}}
+#' @param type String, either "text" or "raw". Passed to `httr::content`
 #'
 #' @family blob store functions
 #' @export
@@ -713,7 +713,7 @@ azureDeleteBlob <- function(azureActiveContext, blob, directory,
 
   SIG <- getSig(azureActiveContext, url = URL, verb = "DELETE", key = storageKey,
                 storageAccount = storageAccount, container = container,
-                CMD = paste0("/", blob), dateS = D1)
+                CMD = paste0("/", blob), dateSig = D1)
 
   AT <- paste0("SharedKey ", storageAccount, ":", SIG)
 

R/AzureContainer.R

@@ -57,7 +57,7 @@ azureListStorageContainers <- function(azureActiveContext, storageAccount, stora
   D1 <- format(Sys.time(), "%a, %d %b %Y %H:%M:%S %Z", tz = "GMT")
 
   SIG <- getSig(azureActiveContext, url = URL, verb = "GET", key = STK, storageAccount = SAI,
-                CMD = "\ncomp:list", dateS = D1)
+                CMD = "\ncomp:list", dateSig = D1)
 
   AT <- paste0("SharedKey ", SAI, ":", SIG)
 
@@ -177,7 +177,7 @@ azureCreateStorageContainer <- function(azureActiveContext, container, storageAc
 
   SIG <- getSig(azureActiveContext, url = URL, verb = "PUT", key = STK,
                 storageAccount = SAI, container = CNTR,
-                CMD = "\nrestype:container", dateS = D1)
+                CMD = "\nrestype:container", dateSig = D1)
 
   AT <- paste0("SharedKey ", SAI, ":", SIG)
   r <- PUT(URL, add_headers(.headers = c(Authorization = AT, `Content-Length` = "0",
@@ -261,7 +261,7 @@ azureDeleteStorageContainer <- function(azureActiveContext, container, storageAc
   azureActiveContext$resourceGroup <- RGI
   SIG <- getSig(azureActiveContext, url = URL, verb = "DELETE", key = STK,
                 storageAccount = SAI,
-                CMD = paste0(CNTR, "\nrestype:container"), dateS = D1)
+                CMD = paste0(CNTR, "\nrestype:container"), dateSig = D1)
 
   AT <- paste0("SharedKey ", SAI, ":", SIG)
 

R/AzureContextObject.R

@@ -7,7 +7,7 @@
 #' @inheritParams setAzureContext
 #' @family azureActiveContext functions
 #'
-#' @seealso \code{\link{setAzureContext}}
+#' @seealso `setAzureContext`
 #' @export
 createAzureContext <- function(tenantID, clientID, authKey){
   azEnv <- new.env(parent = globalenv())
@@ -33,24 +33,24 @@ dumpAzureContext <- function(azureActiveContext){
 
 #' Updates azureActiveContext object.
 #'
-#' Updates the value of an azureActiveContext object, created by \code{\link{createAzureContext}}
+#' Updates the value of an azureActiveContext object, created by `createAzureContext`
 #'
 #' @param azureActiveContext A container used for caching variables used by AzureSMR
 #' @param tenantID The Tenant ID provided during creation of the Active Directory application / service principal
 #' @param clientID The Client ID provided during creation of the Active Directory application / service principal
 #' @param authKey The Authentication Key provided during creation of the Active Directory application / service principal
-#' @param subscriptionID Set the subscriptionID.  This is obtained automatically by \code{\link{azureAuthenticate}} when only a single subscriptionID is available via Active Directory
-#' @param azToken Azure authentication token, obtained by \code{\link{azureAuthenticate}}
+#' @param subscriptionID Set the subscriptionID.  This is obtained automatically by `azureAuthenticate` when only a single subscriptionID is available via Active Directory
+#' @param azToken Azure authentication token, obtained by `azureAuthenticate`
 #' @param resourceGroup Name of the resource group
 #' @param vmName Name of the virtual Machine
 #' @param storageAccount Name of the azure storage account
 #' @param storageKey Storage key associated with storage account
 #' @param blob Blob name
-#' @param clustername Cluster name, used for HDI and Spark clusters. See \code{\link{azureCreateHDI}}
-#' @param sessionID Spark sessionID. See \code{\link{azureSparkCMD}}
+#' @param clustername Cluster name, used for HDI and Spark clusters. See `azureCreateHDI`
+#' @param sessionID Spark sessionID. See `azureSparkCMD`
 #' @param hdiAdmin HDInsight admin username
 #' @param hdiPassword  HDInsight admin password
-#' @param container Storage container name. See \code{\link{azureListStorageContainers}}
+#' @param container Storage container name. See `azureListStorageContainers`
 #' @param kind HDinsight kind: "hadoop","spark" or "pyspark"
 #'
 #' @family azureActiveContext functions

R/AzureHDI.R

@@ -265,22 +265,22 @@ azureResizeHDI <- function(azureActiveContext, clustername, role = "worker",
   RT <- "Request accepted"
   if (toupper(mode) == "SYNC") {
     azureActiveContext$resourceGroup <- RGI
-    writeLines(paste("azureResizeHDI: Request Submitted: ", Sys.time()))
-    writeLines("Accepted(A), Resizing(R), Succeeded(S)")
+    message(paste("azureResizeHDI: Request Submitted: ", Sys.time()))
+    message("Accepted(A), Resizing(R), Succeeded(S)")
     a <- 1
     while (a > 0) {
       rc <- azureListHDI(azureActiveContext, clustername = clustername)
       rc1 <- rc[9, 1]
       if (rc1 == "Running") {
         message("R")
-        writeLines("")
-        writeLines(paste("Finished Resizing Sucessfully: ", Sys.time()))
+        message("")
+        message(paste("Finished Resizing Sucessfully: ", Sys.time()))
         (break)()
       }
 
       if (rc1 == "Error") {
-        writeLines("")
-        writeLines(paste("Error Resizing: ", Sys.time()))
+        message("")
+        message(paste("Error Resizing: ", Sys.time()))
         (break)()
       }
 
@@ -305,9 +305,9 @@ azureResizeHDI <- function(azureActiveContext, clustername, role = "worker",
     }
     # RT <- clusters[12,1]
   }
-  writeLines(paste("Finished: ", Sys.time()))
+  message(paste("Finished: ", Sys.time()))
 
-  return("Done")
+  return(TRUE)
 }
 
 
@@ -613,21 +613,21 @@ azureCreateHDI <- function(azureActiveContext, clustername, location, kind = "sp
   rl <- content(r, "text", encoding = "UTF-8")
   if (toupper(mode) == "SYNC") {
     azureActiveContext$resourceGroup <- RGI
-    writeLines(paste("azureResizeHDI: Request Submitted: ", Sys.time()))
-    writeLines("Runing(C), Succeeded(S)")
+    message(paste("azureResizeHDI: Request Submitted: ", Sys.time()))
+    message("Runing(C), Succeeded(S)")
     a <- 1
     while (a > 0) {
       rc <- azureListHDI(azureActiveContext, clustername = clustername)
       rc1 <- rc[8, 1]
       if (rc1 == "Succeeded") {
         message("S")
-        writeLines("")
-        writeLines(paste("Finished Creating Sucessfully: ", Sys.time()))
+        message("")
+        message(paste("Finished Creating Sucessfully: ", Sys.time()))
         (break)()
       }
       if (rc1 == "Error") {
-        writeLines("")
-        writeLines(paste("Error Creating: ", Sys.time()))
+        message("")
+        message(paste("Error Creating: ", Sys.time()))
         (break)()
       }
       a <- a + 1
@@ -644,8 +644,8 @@ azureCreateHDI <- function(azureActiveContext, clustername, location, kind = "sp
   azureActiveContext$hdiAdmin <- adminUser
   azureActiveContext$hdiPassword  <- adminPassword
   azureActiveContext$clustername <- clustername
-  writeLines(paste("Finished: ", Sys.time()))
-  return("Done")}
+  message(paste("Finished: ", Sys.time()))
+  return(TRUE)}
 
 
 #' Run Script Action on HDI Cluster.

R/AzureHive.R

@@ -121,8 +121,8 @@ azureHiveSQL <- function(azureActiveContext, CMD, clustername, hdiAdmin,
   rl <- content(r, "text", encoding = "UTF-8")
   df <- fromJSON(rl)
 
-  writeLines(paste("CMD Running: ", Sys.time()))
-  writeLines("Prep(P), Running(R), Completed(C)")
+  message(paste("CMD Running: ", Sys.time()))
+  message("Prep(P), Running(R), Completed(C)")
   DUR <- 2
   # print(df$status$state)
   while (df$status$state == "RUNNING" | df$status$state == "PREP") {

R/AzureSM-package.R → R/AzureSMR-package.R

@@ -4,12 +4,10 @@
 #'
 #' This enables you to use and change many Azure resources, including:
 #'
-#' \itemize{
-#' \item Storage blobs
-#' \item HDInsight (Nodes, Hive, Spark)
-#' \item Azure Resource Manager
-#' \item Virtual Machines
-#' }
+#' * Storage blobs
+#' * HDInsight (Nodes, Hive, Spark)
+#' * Azure Resource Manager
+#' * Virtual Machines
 #'
 #'
 #' @name AzureSMR-package

R/AzureSpark.R

@@ -191,7 +191,7 @@ azureSparkStopSession <- function(azureActiveContext, clustername, hdiAdmin,
 
   if (status_code(r) != "200")
     stop(paste("Error Return Code:", status_code(r)))
-  return("Done")
+  return(TRUE)
 }
 
 
@@ -275,8 +275,8 @@ azureSparkCMD <- function(azureActiveContext, CMD, clustername, hdiAdmin,
   URL <- paste("https://", CN, ".azurehdinsight.net/livy/", rh$location,
                sep = "")
   # print(URL)
-  writeLines(paste("CMD Running: ", Sys.time()))
-  writeLines("Running(R), Completed(C)")
+  message(paste("CMD Running: ", Sys.time()))
+  message("Running(R), Completed(C)")
 
   while (df$state == "running") {
     Sys.sleep(DUR)
@@ -373,8 +373,8 @@ azureSparkJob <- function(azureActiveContext, FILE, clustername, hdiAdmin,
   URL <- paste("https://", CN, ".azurehdinsight.net/livy/batches/", BI,
                sep = "")
   # print(URL)
-  writeLines(paste("CMD Running: ", Sys.time()))
-  writeLines("Running(R), Completed(C)")
+  message(paste("CMD Running: ", Sys.time()))
+  message("Running(R), Completed(C)")
   LOGURL2 <- ""
 
   while (df$state == "running") {
@@ -392,8 +392,8 @@ azureSparkJob <- function(azureActiveContext, FILE, clustername, hdiAdmin,
   }
   message("C")
   STATE <- df$state
-  writeLines("")
-  writeLines(paste("Finished Running statement: ", Sys.time()))
+  message("")
+  message(paste("Finished Running statement: ", Sys.time()))
 
   # BID = gsub('application_','container_',df$appId) print(df$log[2])
   # HN<- strsplit(df$log[2], ' ')