From 2b4175986d908477f1a33ed538036c29ef4fc28f Mon Sep 17 00:00:00 2001 From: Brian Hoang Date: Mon, 12 Jun 2017 09:09:19 -0700 Subject: [PATCH 01/20] Added initial commit for Azure resource management for batch --- NAMESPACE | 3 + R/AzureBatch.R | 138 +++++++++++++++++++++++++++++++++ man/azureBatchGetKey.Rd | 25 ++++++ man/azureCreateBatchAccount.Rd | 30 +++++++ man/azureDeleteBatchAccount.Rd | 25 ++++++ 5 files changed, 221 insertions(+) create mode 100644 R/AzureBatch.R create mode 100644 man/azureBatchGetKey.Rd create mode 100644 man/azureCreateBatchAccount.Rd create mode 100644 man/azureDeleteBatchAccount.Rd diff --git a/NAMESPACE b/NAMESPACE index c553922..9ef9084 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,15 +6,18 @@ S3method(summary,azureScriptActionHistory) export(AzureListRG) export(as.azureActiveContext) export(azureAuthenticate) +export(azureBatchGetKey) export(azureBlobCD) export(azureBlobFind) export(azureBlobLS) export(azureCancelDeploy) export(azureCheckToken) +export(azureCreateBatchAccount) export(azureCreateHDI) export(azureCreateResourceGroup) export(azureCreateStorageAccount) export(azureCreateStorageContainer) +export(azureDeleteBatchAccount) export(azureDeleteBlob) export(azureDeleteDeploy) export(azureDeleteHDI) diff --git a/R/AzureBatch.R b/R/AzureBatch.R new file mode 100644 index 0000000..44af6a8 --- /dev/null +++ b/R/AzureBatch.R @@ -0,0 +1,138 @@ + +#' Create an Azure Batch Account. +#' +#' @inheritParams setAzureContext +#' @inheritParams azureAuthenticate +#' @inheritParams azureBatchGetKey +#' @param location A string for the location to create batch account +#' @param asynchronous If TRUE, submits asynchronous request to Azure. Otherwise waits until batch account is created. +#' @family Batch account functions +#' @export +azureCreateBatchAccount <- function(azureActiveContext, batchAccount, + location = "northeurope", + resourceGroup, subscriptionID, + asynchronous = FALSE, verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) + azureCheckToken(azureActiveContext) + azToken <- azureActiveContext$Token + + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup + assert_that(is_resource_group(resourceGroup)) + assert_that(is_subscription_id(subscriptionID)) + assert_that(is_storage_account(batchAccount)) + + verbosity <- set_verbosity(verbose) + + bodyI <- paste0('{ + "location":"', location, '", + }' + ) + + URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + "/resourceGroups/", resourceGroup, "/providers/Microsoft.Batch/batchAccounts/", + batchAccount, "?api-version=2017-05-01") + + r <- PUT(URL, azureApiHeaders(azToken), body = bodyI, encode = "json", verbosity) + + if (status_code(r) == 409) { + message("409: Conflict : Account already exists with the same name") + return(TRUE) + } + + if (status_code(r) == 200) { + message("Account already exists with the same properties") + } + stopWithAzureError(r) + + rl <- content(r, "text", encoding = "UTF-8") + azureActiveContext$batchAccount <- batchAccount + azureActiveContext$resourceGroup <- resourceGroup + message("Create request Accepted. It can take a few moments to provision the batch account") + + if (!asynchronous) { + wait_for_azure( + batchAccount %in% azureListSA(azureActiveContext)$batchAccount + ) + } + TRUE +} + +#' Delete an Azure Batch Account. +#' +#' @inheritParams setAzureContext +#' @inheritParams azureAuthenticate +#' @inheritParams azureSAGetKey + +#' @family Batch account functions +#' @export +azureDeleteBatchAccount <- function(azureActiveContext, batchAccount, + resourceGroup, subscriptionID, verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) + azureCheckToken(azureActiveContext) + azToken <- azureActiveContext$Token + + if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + + assert_that(is_storage_account(batchKey)) + assert_that(is_resource_group(resourceGroup)) + assert_that(is_subscription_id(subscriptionID)) + verbosity <- set_verbosity(verbose) + + URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + "/resourceGroups/", resourceGroup, "/providers/Microsoft.Batch/batchAccounts/", + batchAccount, "?api-version=2017-05-01") + + r <- DELETE(URL, azureApiHeaders(azToken), verbosity) + + if (status_code(r) == 204) { + warning("Batch Account not found") + return(FALSE) + } + if (status_code(r) != 200) stopWithAzureError(r) + + azureActiveContext$batchAccount <- batchAccount + azureActiveContext$resourceGroup <- resourceGroup + TRUE +} + +#' Get the Batch Keys for Specified Batch Account. +#' +#' @inheritParams setAzureContext +#' @inheritParams azureAuthenticate +#' +#' @family Batch account functions +#' @export +azureBatchGetKey <- function(azureActiveContext, batchAccount, + resourceGroup, subscriptionID, verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) + azureCheckToken(azureActiveContext) + azToken <- azureActiveContext$Token + + if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + + assert_that(is_storage_account(storageAccount)) + assert_that(is_resource_group(resourceGroup)) + assert_that(is_subscription_id(subscriptionID)) + verbosity <- set_verbosity(verbose) + + message("Fetching Storage Key..") + + URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + "/resourceGroups/", resourceGroup, + "/providers/Microsoft.Batch/batchAccounts/", batchAccount, + "/listkeys?api-version=2017-05-01") + + r <- POST(URL, azureApiHeaders(azToken), verbosity) + stopWithAzureError(r) + + rl <- content(r, "text", encoding = "UTF-8") + df <- fromJSON(rl) + azureActiveContext$batchAccount <- batchAccount + azureActiveContext$resourceGroup <- resourceGroup + azureActiveContext$batchKey <- df$keys$value[1] + + return(azureActiveContext$batchKey) +} \ No newline at end of file diff --git a/man/azureBatchGetKey.Rd b/man/azureBatchGetKey.Rd new file mode 100644 index 0000000..c1283d9 --- /dev/null +++ b/man/azureBatchGetKey.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureBatch.R +\name{azureBatchGetKey} +\alias{azureBatchGetKey} +\title{Get the Batch Keys for Specified Batch Account.} +\usage{ +azureBatchGetKey(azureActiveContext, batchAccount, resourceGroup, + subscriptionID, verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{resourceGroup}{Name of the resource group} + +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} + +\item{verbose}{Print Tracing information (Default False)} +} +\description{ +Get the Batch Keys for Specified Batch Account. +} +\seealso{ +Other Batch account functions: \code{\link{azureCreateBatchAccount}}, + \code{\link{azureDeleteBatchAccount}} +} diff --git a/man/azureCreateBatchAccount.Rd b/man/azureCreateBatchAccount.Rd new file mode 100644 index 0000000..d605a50 --- /dev/null +++ b/man/azureCreateBatchAccount.Rd @@ -0,0 +1,30 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureBatch.R +\name{azureCreateBatchAccount} +\alias{azureCreateBatchAccount} +\title{Create an Azure Batch Account.} +\usage{ +azureCreateBatchAccount(azureActiveContext, batchAccount, + location = "northeurope", resourceGroup, subscriptionID, + asynchronous = FALSE, verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{location}{A string for the location to create batch account} + +\item{resourceGroup}{Name of the resource group} + +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} + +\item{asynchronous}{If TRUE, submits asynchronous request to Azure. Otherwise waits until batch account is created.} + +\item{verbose}{Print Tracing information (Default False)} +} +\description{ +Create an Azure Batch Account. +} +\seealso{ +Other Batch account functions: \code{\link{azureBatchGetKey}}, + \code{\link{azureDeleteBatchAccount}} +} diff --git a/man/azureDeleteBatchAccount.Rd b/man/azureDeleteBatchAccount.Rd new file mode 100644 index 0000000..af204c2 --- /dev/null +++ b/man/azureDeleteBatchAccount.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureBatch.R +\name{azureDeleteBatchAccount} +\alias{azureDeleteBatchAccount} +\title{Delete an Azure Batch Account.} +\usage{ +azureDeleteBatchAccount(azureActiveContext, batchAccount, resourceGroup, + subscriptionID, verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{resourceGroup}{Name of the resource group} + +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} + +\item{verbose}{Print Tracing information (Default False)} +} +\description{ +Delete an Azure Batch Account. +} +\seealso{ +Other Batch account functions: \code{\link{azureBatchGetKey}}, + \code{\link{azureCreateBatchAccount}} +} From 62a6585d5b85493dab702aecdd8b2783b8289b74 Mon Sep 17 00:00:00 2001 From: Brian Date: Mon, 12 Jun 2017 12:08:41 -0700 Subject: [PATCH 02/20] Added List Batch Accounts function --- NAMESPACE | 1 + R/AzureBatch.R | 41 ++++++++++++++++++++++++++++++------ man/azureBatchGetKey.Rd | 8 ++++--- man/azureListBatchAccount.Rd | 27 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 man/azureListBatchAccount.Rd diff --git a/NAMESPACE b/NAMESPACE index 9ef9084..6803dbc 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -33,6 +33,7 @@ export(azureHDIConf) export(azureHiveSQL) export(azureHiveStatus) export(azureListAllResources) +export(azureListBatchAccount) export(azureListHDI) export(azureListRG) export(azureListSA) diff --git a/R/AzureBatch.R b/R/AzureBatch.R index 44af6a8..f601380 100644 --- a/R/AzureBatch.R +++ b/R/AzureBatch.R @@ -1,3 +1,32 @@ +#' List batch accounts. +#' +#' @inheritParams setAzureContext +#' @inheritParams azureAuthenticate +#' @inheritParams azureBatchGetKey + +#' @family Batch account functions +#' @export +azureListBatchAccount <- function(azureActiveContext, resourceGroup, subscriptionID, + verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) + azureCheckToken(azureActiveContext) + azToken <- azureActiveContext$Token + + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + assert_that(is_subscription_id(subscriptionID)) + verbosity <- set_verbosity(verbose) + + type_batch <- "Microsoft.Batch/batchAccounts" + + z <- if(missing(resourceGroup)) { + azureListAllResources(azureActiveContext, type = type_batch) + } else { + azureListAllResources(azureActiveContext, type = type_batch, resourceGroup = resourceGroup, subscriptionID = subscriptionID) + } + rownames(z) <- NULL + z$batchAccount <- extractStorageAccount(z$id) + z +} #' Create an Azure Batch Account. #' @@ -52,7 +81,7 @@ azureCreateBatchAccount <- function(azureActiveContext, batchAccount, if (!asynchronous) { wait_for_azure( - batchAccount %in% azureListSA(azureActiveContext)$batchAccount + batchAccount %in% azureListBatchAccount(azureActiveContext, subscriptionID = subscriptionID)$name ) } TRUE @@ -62,7 +91,7 @@ azureCreateBatchAccount <- function(azureActiveContext, batchAccount, #' #' @inheritParams setAzureContext #' @inheritParams azureAuthenticate -#' @inheritParams azureSAGetKey +#' @inheritParams azureBatchGetKey #' @family Batch account functions #' @export @@ -75,7 +104,7 @@ azureDeleteBatchAccount <- function(azureActiveContext, batchAccount, if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID - assert_that(is_storage_account(batchKey)) + assert_that(is_storage_account(batchAccount)) assert_that(is_resource_group(resourceGroup)) assert_that(is_subscription_id(subscriptionID)) verbosity <- set_verbosity(verbose) @@ -113,12 +142,12 @@ azureBatchGetKey <- function(azureActiveContext, batchAccount, if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID - assert_that(is_storage_account(storageAccount)) + assert_that(is_storage_account(batchAccount)) assert_that(is_resource_group(resourceGroup)) assert_that(is_subscription_id(subscriptionID)) verbosity <- set_verbosity(verbose) - message("Fetching Storage Key..") + message("Fetching Batch Key..") URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, "/resourceGroups/", resourceGroup, @@ -132,7 +161,7 @@ azureBatchGetKey <- function(azureActiveContext, batchAccount, df <- fromJSON(rl) azureActiveContext$batchAccount <- batchAccount azureActiveContext$resourceGroup <- resourceGroup - azureActiveContext$batchKey <- df$keys$value[1] + azureActiveContext$batchKey <- df$primary return(azureActiveContext$batchKey) } \ No newline at end of file diff --git a/man/azureBatchGetKey.Rd b/man/azureBatchGetKey.Rd index c1283d9..db77630 100644 --- a/man/azureBatchGetKey.Rd +++ b/man/azureBatchGetKey.Rd @@ -8,11 +8,11 @@ azureBatchGetKey(azureActiveContext, batchAccount, resourceGroup, subscriptionID, verbose = FALSE) } \arguments{ -\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} +\item{azureActiveContext}{A container used for caching variables used by `AzureSMR`} \item{resourceGroup}{Name of the resource group} -\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} +\item{subscriptionID}{Subscription ID. This is obtained automatically by [azureAuthenticate()] when only a single subscriptionID is available via Active Directory} \item{verbose}{Print Tracing information (Default False)} } @@ -21,5 +21,7 @@ Get the Batch Keys for Specified Batch Account. } \seealso{ Other Batch account functions: \code{\link{azureCreateBatchAccount}}, - \code{\link{azureDeleteBatchAccount}} + \code{\link{azureDeleteBatchAccount}}, + \code{\link{azureListBatchAccount}} } + diff --git a/man/azureListBatchAccount.Rd b/man/azureListBatchAccount.Rd new file mode 100644 index 0000000..0347df1 --- /dev/null +++ b/man/azureListBatchAccount.Rd @@ -0,0 +1,27 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureBatch.R +\name{azureListBatchAccount} +\alias{azureListBatchAccount} +\title{List batch accounts.} +\usage{ +azureListBatchAccount(azureActiveContext, resourceGroup, subscriptionID, + verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by `AzureSMR`} + +\item{resourceGroup}{Name of the resource group} + +\item{subscriptionID}{Subscription ID. This is obtained automatically by [azureAuthenticate()] when only a single subscriptionID is available via Active Directory} + +\item{verbose}{Print Tracing information (Default False)} +} +\description{ +List batch accounts. +} +\seealso{ +Other Batch account functions: \code{\link{azureBatchGetKey}}, + \code{\link{azureCreateBatchAccount}}, + \code{\link{azureDeleteBatchAccount}} +} + From 96119cb56701a4bdc8cd47d4202c51b016af47b6 Mon Sep 17 00:00:00 2001 From: Brian Date: Mon, 12 Jun 2017 12:32:20 -0700 Subject: [PATCH 03/20] Renamed list batch accounts function --- NAMESPACE | 2 +- R/AzureBatch.R | 2 +- man/azureBatchGetKey.Rd | 2 +- man/azureCreateBatchAccount.Rd | 8 +++++--- man/azureDeleteBatchAccount.Rd | 8 +++++--- ...azureListBatchAccount.Rd => azureListBatchAccounts.Rd} | 6 +++--- 6 files changed, 16 insertions(+), 12 deletions(-) rename man/{azureListBatchAccount.Rd => azureListBatchAccounts.Rd} (84%) diff --git a/NAMESPACE b/NAMESPACE index 6803dbc..b6c9142 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -33,7 +33,7 @@ export(azureHDIConf) export(azureHiveSQL) export(azureHiveStatus) export(azureListAllResources) -export(azureListBatchAccount) +export(azureListBatchAccounts) export(azureListHDI) export(azureListRG) export(azureListSA) diff --git a/R/AzureBatch.R b/R/AzureBatch.R index f601380..182ccf4 100644 --- a/R/AzureBatch.R +++ b/R/AzureBatch.R @@ -6,7 +6,7 @@ #' @family Batch account functions #' @export -azureListBatchAccount <- function(azureActiveContext, resourceGroup, subscriptionID, +azureListBatchAccounts <- function(azureActiveContext, resourceGroup, subscriptionID, verbose = FALSE) { assert_that(is.azureActiveContext(azureActiveContext)) azureCheckToken(azureActiveContext) diff --git a/man/azureBatchGetKey.Rd b/man/azureBatchGetKey.Rd index db77630..480eb52 100644 --- a/man/azureBatchGetKey.Rd +++ b/man/azureBatchGetKey.Rd @@ -22,6 +22,6 @@ Get the Batch Keys for Specified Batch Account. \seealso{ Other Batch account functions: \code{\link{azureCreateBatchAccount}}, \code{\link{azureDeleteBatchAccount}}, - \code{\link{azureListBatchAccount}} + \code{\link{azureListBatchAccounts}} } diff --git a/man/azureCreateBatchAccount.Rd b/man/azureCreateBatchAccount.Rd index d605a50..ae6cc53 100644 --- a/man/azureCreateBatchAccount.Rd +++ b/man/azureCreateBatchAccount.Rd @@ -9,13 +9,13 @@ azureCreateBatchAccount(azureActiveContext, batchAccount, asynchronous = FALSE, verbose = FALSE) } \arguments{ -\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} +\item{azureActiveContext}{A container used for caching variables used by `AzureSMR`} \item{location}{A string for the location to create batch account} \item{resourceGroup}{Name of the resource group} -\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} +\item{subscriptionID}{Subscription ID. This is obtained automatically by [azureAuthenticate()] when only a single subscriptionID is available via Active Directory} \item{asynchronous}{If TRUE, submits asynchronous request to Azure. Otherwise waits until batch account is created.} @@ -26,5 +26,7 @@ Create an Azure Batch Account. } \seealso{ Other Batch account functions: \code{\link{azureBatchGetKey}}, - \code{\link{azureDeleteBatchAccount}} + \code{\link{azureDeleteBatchAccount}}, + \code{\link{azureListBatchAccounts}} } + diff --git a/man/azureDeleteBatchAccount.Rd b/man/azureDeleteBatchAccount.Rd index af204c2..45ce7fd 100644 --- a/man/azureDeleteBatchAccount.Rd +++ b/man/azureDeleteBatchAccount.Rd @@ -8,11 +8,11 @@ azureDeleteBatchAccount(azureActiveContext, batchAccount, resourceGroup, subscriptionID, verbose = FALSE) } \arguments{ -\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} +\item{azureActiveContext}{A container used for caching variables used by `AzureSMR`} \item{resourceGroup}{Name of the resource group} -\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} +\item{subscriptionID}{Subscription ID. This is obtained automatically by [azureAuthenticate()] when only a single subscriptionID is available via Active Directory} \item{verbose}{Print Tracing information (Default False)} } @@ -21,5 +21,7 @@ Delete an Azure Batch Account. } \seealso{ Other Batch account functions: \code{\link{azureBatchGetKey}}, - \code{\link{azureCreateBatchAccount}} + \code{\link{azureCreateBatchAccount}}, + \code{\link{azureListBatchAccounts}} } + diff --git a/man/azureListBatchAccount.Rd b/man/azureListBatchAccounts.Rd similarity index 84% rename from man/azureListBatchAccount.Rd rename to man/azureListBatchAccounts.Rd index 0347df1..7943299 100644 --- a/man/azureListBatchAccount.Rd +++ b/man/azureListBatchAccounts.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/AzureBatch.R -\name{azureListBatchAccount} -\alias{azureListBatchAccount} +\name{azureListBatchAccounts} +\alias{azureListBatchAccounts} \title{List batch accounts.} \usage{ -azureListBatchAccount(azureActiveContext, resourceGroup, subscriptionID, +azureListBatchAccounts(azureActiveContext, resourceGroup, subscriptionID, verbose = FALSE) } \arguments{ From b421622006e82c811396af395abfc0cfc12d0f7f Mon Sep 17 00:00:00 2001 From: Brian Date: Mon, 12 Jun 2017 13:57:09 -0700 Subject: [PATCH 04/20] Replace calling with call_azure_sm function --- R/AzureBatch.R | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/R/AzureBatch.R b/R/AzureBatch.R index 182ccf4..596b580 100644 --- a/R/AzureBatch.R +++ b/R/AzureBatch.R @@ -51,18 +51,17 @@ azureCreateBatchAccount <- function(azureActiveContext, batchAccount, assert_that(is_subscription_id(subscriptionID)) assert_that(is_storage_account(batchAccount)) - verbosity <- set_verbosity(verbose) - - bodyI <- paste0('{ + body <- paste0('{ "location":"', location, '", }' ) - URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + uri <- paste0("https://management.azure.com/subscriptions/", subscriptionID, "/resourceGroups/", resourceGroup, "/providers/Microsoft.Batch/batchAccounts/", batchAccount, "?api-version=2017-05-01") - r <- PUT(URL, azureApiHeaders(azToken), body = bodyI, encode = "json", verbosity) + r <- call_azure_sm(azureActiveContext, uri = uri, body = body, + verb = "PUT", verbose = verbose) if (status_code(r) == 409) { message("409: Conflict : Account already exists with the same name") @@ -81,7 +80,7 @@ azureCreateBatchAccount <- function(azureActiveContext, batchAccount, if (!asynchronous) { wait_for_azure( - batchAccount %in% azureListBatchAccount(azureActiveContext, subscriptionID = subscriptionID)$name + batchAccount %in% azureListBatchAccounts(azureActiveContext, subscriptionID = subscriptionID)$name ) } TRUE @@ -107,13 +106,13 @@ azureDeleteBatchAccount <- function(azureActiveContext, batchAccount, assert_that(is_storage_account(batchAccount)) assert_that(is_resource_group(resourceGroup)) assert_that(is_subscription_id(subscriptionID)) - verbosity <- set_verbosity(verbose) - URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + uri <- paste0("https://management.azure.com/subscriptions/", subscriptionID, "/resourceGroups/", resourceGroup, "/providers/Microsoft.Batch/batchAccounts/", batchAccount, "?api-version=2017-05-01") - r <- DELETE(URL, azureApiHeaders(azToken), verbosity) + r <- call_azure_sm(azureActiveContext, uri = uri, + verb = "DELETE", verbose = verbose) if (status_code(r) == 204) { warning("Batch Account not found") @@ -145,16 +144,16 @@ azureBatchGetKey <- function(azureActiveContext, batchAccount, assert_that(is_storage_account(batchAccount)) assert_that(is_resource_group(resourceGroup)) assert_that(is_subscription_id(subscriptionID)) - verbosity <- set_verbosity(verbose) - + message("Fetching Batch Key..") - URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + uri <- paste0("https://management.azure.com/subscriptions/", subscriptionID, "/resourceGroups/", resourceGroup, "/providers/Microsoft.Batch/batchAccounts/", batchAccount, "/listkeys?api-version=2017-05-01") - r <- POST(URL, azureApiHeaders(azToken), verbosity) + r <- call_azure_sm(azureActiveContext, uri = uri, + verb = "POST", verbose = verbose) stopWithAzureError(r) rl <- content(r, "text", encoding = "UTF-8") From 8e841e13027aa49fe957da95bd40e7243f492465 Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Wed, 14 Jun 2017 10:05:38 +0800 Subject: [PATCH 05/20] Default config file loading path portable to Windows OS --- R/zzz.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/zzz.R b/R/zzz.R index 6c8ef31..3c604e3 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,4 +1,6 @@ -AzureSMR.config.default <- "~/.azuresmr/config.json" +AzureSMR.config.default <- ifelse(Sys.info()["sysname"] == "Windows", + paste0("C:/Users/", Sys.getenv("USERNAME"), "/.azuresmr/config.json"), + "~/.azuresmr/config.json") .onAttach <- function(libname, pkgname) { if (is.null(getOption("AzureSMR.config"))) From 2ecda6615a374a757333b65cfd66bb7ae74b1810 Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Wed, 14 Jun 2017 11:29:01 +0800 Subject: [PATCH 06/20] Unit tests for cost and pricing functions --- tests/testthat/test-cost.R | 103 +++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tests/testthat/test-cost.R diff --git a/tests/testthat/test-cost.R b/tests/testthat/test-cost.R new file mode 100644 index 0000000..a308ea5 --- /dev/null +++ b/tests/testthat/test-cost.R @@ -0,0 +1,103 @@ +# ----------------------------------------------------------------- +# Test for cost functions. +# ----------------------------------------------------------------- + +# preambles. + +if (interactive()) library("testthat") + +settingsfile <- getOption("AzureSMR.config") +config <- read.AzureSMR.config() + +# setup. + +context("Data consumption and cost") + +asc <- createAzureContext() +with(config, + setAzureContext(asc, tenantID=tenantID, clientID=clientID, authKey=authKey) +) +azureAuthenticate(asc) + +timestamp <- format(Sys.time(), format="%y%m%d%H%M") +resourceGroup_name <- paste0("AzureSMtest_", timestamp) +sa_name <- paste0("azuresmr", timestamp) + +# run test. + +# get cost by day. + +test_that("Get cost by day", { + skip_if_missing_config(settingsfile) + + time_end <- paste0(as.Date(Sys.Date()), "00:00:00") + time_start <- paste0(as.Date(Sys.Date() - 365), "00:00:00") + + res <- azureDataConsumption(azureActiveContext=asc, + timeStart=time_start, + timeEnd=time_end, + granularity="Daily") + + expect_type(res, type="list") + expect_identical(object=names(res), expected=c("usageStartTime", + "usageEndTime", + "meterName", + "meterCategory", + "meterSubCategory", + "unit", + "meterId", + "quantity", + "meterRegion")) +}) + +# get pricing rates for meters under subscription. + +test_that("Get pricing rates", { + skip_if_missing_config(settingsfile) + + res <- azurePricingRates(azureActiveContext=asc, + currency=config$CURRENCY, + locale=config$LOCALE, + offerId=config$OFFER, + region=config$REGION) + + expect_type(res, type="list") + expect_identical(object=names(res), expected=c("EffectiveDate", + "IncludedQuantity", + "MeterCategory", + "MeterId", + "MeterName", + "MeterRegion", + "MeterStatus", + "MeterSubCategory", + "Unit", + "MeterRate")) +}) + + +# total expense by day. + +test_that("Get cost by day", { + skip_if_missing_config(settingsfile) + + time_end <- paste0(as.Date(Sys.Date()), "00:00:00") + time_start <- paste0(as.Date(Sys.Date() - 365), "00:00:00") + + res <- azureExpenseCalculator(azureActiveContext=asc, + timeStart=time_start, + timeEnd=time_end, + granularity="Daily", + currency=config$CURRENCY, + locale=config$LOCALE, + offerId=config$OFFER, + region=config$REGION) + + expect_type(res, type="list") + expect_identical(object=names(res), expected=c("MeterName", + "MeterCategory", + "MeterSubCategory", + "totalQuantity", + "Unit", + "MeterRate", + "Cost")) +}) \ No newline at end of file From e7a852d995e0f4d3b3c7f5125a3fd40cf0876a7c Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Wed, 14 Jun 2017 11:29:23 +0800 Subject: [PATCH 07/20] Add imports of dplyr, magrittr, and lubridate --- R/AzureSMR-package.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R/AzureSMR-package.R b/R/AzureSMR-package.R index 81111a0..d73bae2 100644 --- a/R/AzureSMR-package.R +++ b/R/AzureSMR-package.R @@ -42,5 +42,9 @@ #' @importFrom httr add_headers headers content status_code http_status authenticate #' @importFrom httr GET PUT DELETE POST #' @importFrom XML htmlParse xpathApply xpathSApply xmlValue +#' +#' @import dplyr +#' @import magrittr +#' @importFrom lubridate hour minute second #' NULL From 248c63b104ee7b0633dcb69dd8442dc934eaa8cb Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Wed, 14 Jun 2017 11:29:39 +0800 Subject: [PATCH 08/20] Added functions for data consumption, pricing, and cost calculation. --- R/AzureCost.R | 357 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 357 insertions(+) create mode 100644 R/AzureCost.R diff --git a/R/AzureCost.R b/R/AzureCost.R new file mode 100644 index 0000000..021e449 --- /dev/null +++ b/R/AzureCost.R @@ -0,0 +1,357 @@ +#' @title Get data consumption of an Azure subscription for a time period. Aggregation method can be either daily based or hourly based. +#' +#' @note Formats of start time point and end time point follow ISO 8601 standard. Say if one would like to calculate data consumption between Feb 21, 2017 to Feb 25, 2017, with an aggregation granularity of "daily based", the inputs should be "2017-02-21 00:00:00" and "2017-02-25 00:00:00", for start time point and end time point, respectively. If the aggregation granularity is hourly based, the inputs can be "2017-02-21 01:00:00" and "2017-02-21 02:00:00", for start and end time point, respectively. NOTE by default the Azure data consumption API does not allow an aggregation granularity that is finer than an hour. In the case of "hourly based" granularity, if the time difference between start and end time point is less than an hour, data consumption will still be calculated hourly based with end time postponed. For example, if the start time point and end time point are "2017-02-21 00:00:00" and "2017-02-21 00:45:00", the actual returned results are are data consumption in the interval of "2017-02-21 00:00:00" and "2017-02-21 01:00:00". However this calculation is merely for retrieving the information of an existing DSVM instance (e.g., meterId) with which the pricing rate is multiplied by to obtain the overall expense. Time zone of all time inputs are synchronized to UTC. +#' +#' @param azureActiveContext AzureSMR context object. +#' +#' @param instance Instance of Azure DSVM name that one would like to check expense. It is by default empty, which returns data consumption for all instances under subscription. +#' +#' @param timeStart Start time. +#' +#' @param timeEnd End time. +#' +#' @param granularity Aggregation granularity. Can be either "Daily" or "Hourly". +#' @export +azureDataConsumption <- function(azureActiveContext, + instance="", + timeStart, + timeEnd, + granularity="Hourly", + verbose=FALSE) { + + # check the validity of credentials. + + assert_that(is.azureActiveContext(azureActiveContext)) + + # renew token if it expires. + + azureCheckToken(azureActiveContext) + + # preconditions here... + + if(missing(timeStart)) + stop("Please specify a starting time point in YYYY-MM-DD HH:MM:SS format.") + + if(missing(timeEnd)) + stop("Please specify an ending time point in YYYY-MM-DD HH:MM:SS format.") + + ds <- try(as.POSIXlt(timeStart, format= "%Y-%m-%d %H:%M:%S", tz="UTC")) + de <- try(as.POSIXlt(timeEnd, format= "%Y-%m-%d %H:%M:%S", tz="UTC")) + + if (class(ds) == "try-error" || + is.na(ds) || + class(de) == "try-error" || + is.na(de)) + stop("Input date format should be YYYY-MM-DD HH:MM:SS.") + + timeStart <- ds + timeEnd <- de + + if (timeStart >= timeEnd) + stop("End time is no later than start time!") + + lubridate::minute(timeStart) <- 0 + lubridate::second(timeStart) <- 0 + lubridate::minute(timeEnd) <- 0 + lubridate::second(timeEnd) <- 0 + + if (granularity == "Daily") { + + # timeStart and timeEnd should be some day at midnight. + + lubridate::hour(timeStart) <- 0 + lubridate::hour(timeEnd) <- 0 + + } + + # If the computation time is less than a hour, timeEnd will be incremented by an hour to get the total cost within an hour aggregated from timeStart. However, only the consumption on computation is considered in the returned data, and the computation consumption will then be replaced with the actual timeEnd - timeStart. + + # NOTE: estimation of cost in this case is rough though, it captures the major component of total cost, which originates from running an Azure instance. Other than computation cost, there are also cost on activities such as data transfer, software library license, etc. This is not included in the approximation here until a solid method for capturing those consumption data is found. Data ingress does not generate cost, but data egress does. Usually the occurrence of data transfer is not that frequent as computation, and pricing rates for data transfer is also less than computation (e.g., price rate of "data transfer in" is ~ 40% of that of computation on an A3 virtual machine). + + # TODO: inlude other types of cost for jobs that take less than an hour. + + if (as.numeric(timeEnd - timeStart) == 0) { + writeLines("Difference between timeStart and timeEnd is less than the aggregation granularity. Cost is estimated solely on computation running time.") + + # increment timeEnd by one hour. + + timeEnd <- timeEnd + 3600 + } + + # reformat time variables to make them compatible with API call. + + START <- URLencode(paste(as.Date(timeStart), "T", + sprintf("%02d", lubridate::hour(timeStart)), ":", sprintf("%02d", lubridate::minute(timeStart)), ":", sprintf("%02d", lubridate::second(timeStart)), "+", + "00:00", + sep=""), + reserved=TRUE) + + END <- URLencode(paste(as.Date(timeEnd), "T", + sprintf("%02d", lubridate::hour(timeEnd)), ":", sprintf("%02d", lubridate::minute(timeEnd)), ":", sprintf("%02d", lubridate::second(timeEnd)), "+", + "00:00", + sep=""), + reserved=TRUE) + + URL <- + sprintf("https://management.azure.com/subscriptions/%s/providers/Microsoft.Commerce/UsageAggregates?api-version=%s&reportedStartTime=%s&reportedEndTime=%s&aggregationgranularity=%s&showDetails=%s", + azureActiveContext$subscriptionID, + "2015-06-01-preview", + START, + END, + granularity, + "false" + ) + + r <- call_azure_sm(azureActiveContext, + uri=URL, + verb="GET", + verbose=verbose) + + stopWithAzureError(r) + + rl <- content(r, "text", encoding="UTF-8") + + df <- fromJSON(rl) + + df_use <- + df$value$properties %>% + select(-infoFields) + + inst_data <- + df$value$properties$instanceData %>% + lapply(., fromJSON) + + # retrieve results that match instance name. + + if (instance != "") { + instance_detect <- function(inst_data) { + return(basename(inst_data$Microsoft.Resources$resourceUri) == instance) + } + + index_instance <- which(unlist(lapply(inst_data, instance_detect))) + + if(!missing(instance)) { + if(length(index_instance) == 0) + stop("No data consumption records found for the instance during the given period.") + df_use <- df_use[index_instance, ] + } else if(missing(instance)) { + if(length(index_resource) == 0) + stop("No data consumption records found for the resource group during the given period.") + df_use <- df_use[index_resource, ] + } + } + + # if time difference is less than one hour. Only return one row of computation consumption whose value is the time difference. + + # timeEnd <- timeEnd - 3600 + + if(as.numeric(timeEnd - timeStart) == 0) { + + time_diff <- as.numeric(de - ds) / 3600 + + df_use %<>% + select(usageStartTime, + usageEndTime, + meterName, + meterCategory, + meterSubCategory, + unit, + meterId, + quantity, + meterRegion) %>% + filter(meterName == "Compute Hours") %>% + filter(row_number() == 1) %>% + mutate(quantity = time_diff) %>% + mutate(usageStartTime = as.POSIXct(usageStartTime)) %>% + mutate(usageEndTime = as.POSIXct(usageEndTime)) + + writeLines(sprintf("The data consumption for %s between %s and %s is", + instance, + as.character(timeStart), + as.character(timeEnd))) + return(df_use) + + } else { + + # NOTE the maximum number of records returned from API is limited to 1000. + + if (nrow(df_use) == 1000 && max(as.POSIXct(df_use$usageEndTime)) < as.POSIXct(END)) { + warning(sprintf("The number of records in the specified time period %s to %s exceeds the limit that can be returned from API call. Consumption information is truncated. Please use a small period instead.", START, END)) + } + + df_use %<>% + select(usageStartTime, + usageEndTime, + meterName, + meterCategory, + meterSubCategory, + unit, + meterId, + quantity, + meterRegion) %>% + mutate(usageStartTime = as.POSIXct(usageStartTime)) %>% + mutate(usageEndTime = as.POSIXct(usageEndTime)) + + writeLines(sprintf("The data consumption for %s between %s and %s is", + instance, + as.character(timeStart), + as.character(timeEnd))) + + df_use + } +} + +#' @title Get pricing details of resources under a subscription. +#' +#' @param azureActiveContext - Azure Context Object. +#' +#' @param currency Currency in which price rating is measured. +#' +#' @param locale Locality information of subscription. +#' +#' @param offerId Offer ID of the subscription. Detailed information can be found at https://azure.microsoft.com/en-us/support/legal/offer-details/ +#' +#' @param region region information about the subscription. +#' +#' @export +azurePricingRates <- function(azureActiveContext, + currency, + locale, + offerId, + region, + verbose=FALSE +) { + # renew token if it expires. + + azureCheckToken(azureActiveContext) + + # preconditions. + + if(missing(currency)) + stop("Error: please provide currency information.") + + if(missing(locale)) + stop("Error: please provide locale information.") + + if(missing(offerId)) + stop("Error: please provide offer ID.") + + if(missing(region)) + stop("Error: please provide region information.") + + url <- paste( + "https://management.azure.com/subscriptions/", azureActiveContext$subscriptionID, + "/providers/Microsoft.Commerce/RateCard?api-version=2016-08-31-preview&$filter=", + "OfferDurableId eq '", offerId, "'", + " and Currency eq '", currency, "'", + " and Locale eq '", locale, "'", + " and RegionInfo eq '", region, "'", + sep="") + + url <- URLencode(url) + + r <- call_azure_sm(azureActiveContext, + uri=url, + verb="GET", + verbose=verbose) + + stopWithAzureError(r) + + # r <- GET(url, add_headers(.headers=c(Authorization=azureActiveContext$Token, "Content-Type"="application/json"))) + + rl <- fromJSON(content(r, "text", encoding="UTF-8"), simplifyDataFrame=TRUE) + + df_meter <- rl$Meters + df_meter$MeterRate <- rl$Meters$MeterRates$`0` + + # an irresponsible drop of MeterRates and MeterTags. Will add them back after having a better handle of them. + + df_meter <- subset(df_meter, select=-MeterRates) + df_meter <- subset(df_meter, select=-MeterTags) + + return(df_meter) +} + +#' @title Calculate cost of using a specific instance of Azure for certain period. +#' +#' @param azureActiveContext AzureSMR context. +#' +#' @param instance Instance of Azure instance that one would like to check expense. No matter whether resource group is given or not, if a instance of instance is given, data consumption of that instance is returned. +#' +#' @param timeStart Start time. +#' +#' @param timeEnd End time. +#' +#' @param granularity Aggregation granularity. Can be either "Daily" or "Hourly". +#' +#' @param currency Currency in which price rating is measured. +#' +#' @param locale Locality information of subscription. +#' +#' @param offerId Offer ID of the subscription. Detailed information can be found at https://azure.microsoft.com/en-us/support/legal/offer-details/ +#' +#' @param region region information about the subscription. +#' +#' @return Total cost measured in the given currency of the specified Azure instance in the period. +#' +#' @note Note if difference between \code{timeStart} and \code{timeEnd} is less than the finest granularity, e.g., "Hourly" (we notice this is a usual case when one needs to be aware of the charges of a job that takes less than an hour), the expense will be estimated based solely on computation hour. That is, the total expense is the multiplication of computation hour and pricing rate of the DSVM instance. +#' +#' @export +azureExpenseCalculator <- function(azureActiveContext, + instance="", + timeStart, + timeEnd, + granularity, + currency, + locale, + offerId, + region, + verbose=FALSE) { + df_use <- + azureDataConsumption(azureActiveContext, + instance=instance, + timeStart=timeStart, + timeEnd=timeEnd, + granularity=granularity, + verbose=verbose) %>% + select(meterId, + meterSubCategory, + usageStartTime, + usageEndTime, + quantity) + + df_used_data <- + group_by(df_use, meterId) %>% + arrange(usageStartTime, usageEndTime) %>% + summarise(usageStartDate=as.Date(min(usageStartTime), tz=Sys.timezone()), + usageEndDate=as.Date(max(usageEndTime), tz=Sys.timezone()), + totalQuantity=sum(quantity)) %>% + ungroup() + + # use meterId to find pricing rates and then calculate total cost. + + df_rates <- azurePricingRates(azureActiveContext, + currency=currency, + locale=locale, + region=region, + offerId=offerId, + verbose=verbose) + + meter_list <- df_used_data$meterId + + df_used_rates <- + filter(df_rates, MeterId %in% meter_list) %>% + rename(meterId=MeterId) + + df_cost <- + left_join(df_used_data, df_used_rates, by="meterId") %>% + mutate(Cost=totalQuantity * MeterRate) %>% + select(-IncludedQuantity, -EffectiveDate, -MeterStatus, -usageStartDate, -usageEndDate, -meterId, -MeterRegion) %>% + na.omit() + + # reorder columns. + + df_cost <- df_cost[, c(3, 2, 4, 1, 5, 6, 7)] + + df_cost +} From 05fea4dcf144677b8752a8ed2afb552eb6ab8a9a Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Wed, 14 Jun 2017 11:30:02 +0800 Subject: [PATCH 09/20] Updated NAMESPACE with new imports of libraries and exported functions --- NAMESPACE | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NAMESPACE b/NAMESPACE index c553922..011517b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -15,6 +15,7 @@ export(azureCreateHDI) export(azureCreateResourceGroup) export(azureCreateStorageAccount) export(azureCreateStorageContainer) +export(azureDataConsumption) export(azureDeleteBlob) export(azureDeleteDeploy) export(azureDeleteHDI) @@ -24,6 +25,7 @@ export(azureDeleteVM) export(azureDeletestorageAccount) export(azureDeployStatus) export(azureDeployTemplate) +export(azureExpenseCalculator) export(azureGetAllVMstatus) export(azureGetBlob) export(azureHDIConf) @@ -40,6 +42,7 @@ export(azureListStorageBlobs) export(azureListStorageContainers) export(azureListSubscriptions) export(azureListVM) +export(azurePricingRates) export(azurePutBlob) export(azureResizeHDI) export(azureRunScriptAction) @@ -60,6 +63,8 @@ export(dumpAzureContext) export(is.azureActiveContext) export(read.AzureSMR.config) export(setAzureContext) +import(dplyr) +import(magrittr) importFrom(DT,dataTableOutput) importFrom(DT,renderDataTable) importFrom(XML,htmlParse) @@ -82,6 +87,9 @@ importFrom(httr,headers) importFrom(httr,http_status) importFrom(httr,status_code) importFrom(jsonlite,fromJSON) +importFrom(lubridate,hour) +importFrom(lubridate,minute) +importFrom(lubridate,second) importFrom(miniUI,gadgetTitleBar) importFrom(miniUI,miniContentPanel) importFrom(miniUI,miniPage) From 8cfe1bf3ebf3a078e23f40d342e0396fea84b289 Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Wed, 14 Jun 2017 11:31:44 +0800 Subject: [PATCH 10/20] man files for the cost functions --- man/azureDataConsumption.Rd | 24 ++++++++++++++++++++++++ man/azureExpenseCalculator.Rd | 35 +++++++++++++++++++++++++++++++++++ man/azurePricingRates.Rd | 20 ++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 man/azureDataConsumption.Rd create mode 100644 man/azureExpenseCalculator.Rd create mode 100644 man/azurePricingRates.Rd diff --git a/man/azureDataConsumption.Rd b/man/azureDataConsumption.Rd new file mode 100644 index 0000000..1c34175 --- /dev/null +++ b/man/azureDataConsumption.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureCost.R +\name{azureDataConsumption} +\alias{azureDataConsumption} +\title{Get data consumption of an Azure subscription for a time period. Aggregation method can be either daily based or hourly based.} +\usage{ +azureDataConsumption(azureActiveContext, instance, timeStart, timeEnd, + granularity = "Hourly") +} +\arguments{ +\item{azureActiveContext}{AzureSMR context object.} + +\item{instance}{Instance of Azure DSVM name that one would like to check expense.} + +\item{timeStart}{Start time.} + +\item{timeEnd}{End time.} + +\item{granularity}{Aggregation granularity. Can be either "Daily" or "Hourly".} +} +\note{ +Formats of start time point and end time point follow ISO 8601 standard. Say if one would like to calculate data consumption between Feb 21, 2017 to Feb 25, 2017, with an aggregation granularity of "daily based", the inputs should be "2017-02-21 00:00:00" and "2017-02-25 00:00:00", for start time point and end time point, respectively. If the aggregation granularity is hourly based, the inputs can be "2017-02-21 01:00:00" and "2017-02-21 02:00:00", for start and end time point, respectively. NOTE by default the Azure data consumption API does not allow an aggregation granularity that is finer than an hour. In the case of "hourly based" granularity, if the time difference between start and end time point is less than an hour, data consumption will still be calculated hourly based with end time postponed. For example, if the start time point and end time point are "2017-02-21 00:00:00" and "2017-02-21 00:45:00", the actual returned results are are data consumption in the interval of "2017-02-21 00:00:00" and "2017-02-21 01:00:00". However this calculation is merely for retrieving the information of an existing DSVM instance (e.g., meterId) with which the pricing rate is multiplied by to obtain the overall expense. Time zone of all time inputs are synchronized to UTC. +} + diff --git a/man/azureExpenseCalculator.Rd b/man/azureExpenseCalculator.Rd new file mode 100644 index 0000000..109f737 --- /dev/null +++ b/man/azureExpenseCalculator.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureCost.R +\name{azureExpenseCalculator} +\alias{azureExpenseCalculator} +\title{Calculate cost of using a specific instance of Azure for certain period.} +\usage{ +azureExpenseCalculator(azureActiveContext, instance, timeStart, timeEnd, + granularity, currency, locale, offerId, region) +} +\arguments{ +\item{azureActiveContext}{AzureSMR context.} + +\item{instance}{Instance of Azure instance that one would like to check expense. No matter whether resource group is given or not, if a instance of instance is given, data consumption of that instance is returned.} + +\item{timeStart}{Start time.} + +\item{timeEnd}{End time.} + +\item{granularity}{Aggregation granularity. Can be either "Daily" or "Hourly".} + +\item{currency}{Currency in which price rating is measured.} + +\item{locale}{Locality information of subscription.} + +\item{offerId}{Offer ID of the subscription. Detailed information can be found at https://azure.microsoft.com/en-us/support/legal/offer-details/} + +\item{region}{region information about the subscription.} +} +\value{ +Total cost measured in the given currency of the specified Azure instance in the period. +} +\note{ +Note if difference between \code{timeStart} and \code{timeEnd} is less than the finest granularity, e.g., "Hourly" (we notice this is a usual case when one needs to be aware of the charges of a job that takes less than an hour), the expense will be estimated based solely on computation hour. That is, the total expense is the multiplication of computation hour and pricing rate of the DSVM instance. +} + diff --git a/man/azurePricingRates.Rd b/man/azurePricingRates.Rd new file mode 100644 index 0000000..b8acd59 --- /dev/null +++ b/man/azurePricingRates.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureCost.R +\name{azurePricingRates} +\alias{azurePricingRates} +\title{Get pricing details of resources under a subscription.} +\usage{ +azurePricingRates(azureActiveContext, currency, locale, offerId, region) +} +\arguments{ +\item{azureActiveContext}{- Azure Context Object.} + +\item{currency}{Currency in which price rating is measured.} + +\item{locale}{Locality information of subscription.} + +\item{offerId}{Offer ID of the subscription. Detailed information can be found at https://azure.microsoft.com/en-us/support/legal/offer-details/} + +\item{region}{region information about the subscription.} +} + From 44887778c1415e420fcc9387f9930ef08f16e36d Mon Sep 17 00:00:00 2001 From: Scott Donohoo Date: Wed, 14 Jun 2017 12:40:43 -0500 Subject: [PATCH 11/20] Fix typos and literal/variable mistakes in building the json for the SQL Hive metastore. --- R/hdi_json.R | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/R/hdi_json.R b/R/hdi_json.R index 9c3db08..c652b5a 100644 --- a/R/hdi_json.R +++ b/R/hdi_json.R @@ -92,17 +92,17 @@ paste0(' hive_json = function(hiveServer, hiveDB, hiveUser, hivePassword) { paste0(' - ",hive-site": { - "javax.jdo.option.ConnectionDrivername": "com.microsoft.sqlserver.jdbc.SQLServerDriver", + ,"hive-site": { + "javax.jdo.option.ConnectionDriverName": "com.microsoft.sqlserver.jdbc.SQLServerDriver", "javax.jdo.option.ConnectionURL":"jdbc:sqlserver://', hiveServer, ';database=', hiveDB, ';encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", - "javax.jdo.option.ConnectionUsername":"', hiveUser, '", + "javax.jdo.option.ConnectionUserName":"', hiveUser, '", "javax.jdo.option.ConnectionPassword":"', hivePassword, '" }, "hive-env": { "hive_database": "Existing MSSQL Server database with SQL authentication", - "hive_database_name": "HIVEDB", + "hive_database_name": "', hiveDB, '", "hive_database_type": "mssql", - "hive_existing_mssql_server_database": "HIVEDB", + "hive_existing_mssql_server_database": "', hiveDB, '", "hive_existing_mssql_server_host":"', hiveServer, '", "hive_hostname":"', hiveServer,'" }' From ea1ef50c0c725cca537579d93949a7b3e371693f Mon Sep 17 00:00:00 2001 From: Andrie de Vries Date: Wed, 14 Jun 2017 19:23:16 +0100 Subject: [PATCH 12/20] Add Azure Batch functions to package documentation; fix some typos --- R/AzureBatch.R | 16 +++++----------- R/AzureSMR-package.R | 5 +++++ man/AzureSMR.Rd | 7 +++++++ man/azureBatchGetKey.Rd | 5 ++--- man/azureCreateBatchAccount.Rd | 9 ++++----- man/azureDeleteBatchAccount.Rd | 9 ++++----- man/azureListBatchAccounts.Rd | 5 ++--- 7 files changed, 29 insertions(+), 27 deletions(-) diff --git a/R/AzureBatch.R b/R/AzureBatch.R index 596b580..0d24165 100644 --- a/R/AzureBatch.R +++ b/R/AzureBatch.R @@ -9,12 +9,9 @@ azureListBatchAccounts <- function(azureActiveContext, resourceGroup, subscriptionID, verbose = FALSE) { assert_that(is.azureActiveContext(azureActiveContext)) - azureCheckToken(azureActiveContext) - azToken <- azureActiveContext$Token if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID assert_that(is_subscription_id(subscriptionID)) - verbosity <- set_verbosity(verbose) type_batch <- "Microsoft.Batch/batchAccounts" @@ -28,7 +25,8 @@ azureListBatchAccounts <- function(azureActiveContext, resourceGroup, subscripti z } -#' Create an Azure Batch Account. + +#' Create an azure batch account. #' #' @inheritParams setAzureContext #' @inheritParams azureAuthenticate @@ -42,8 +40,6 @@ azureCreateBatchAccount <- function(azureActiveContext, batchAccount, resourceGroup, subscriptionID, asynchronous = FALSE, verbose = FALSE) { assert_that(is.azureActiveContext(azureActiveContext)) - azureCheckToken(azureActiveContext) - azToken <- azureActiveContext$Token if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup @@ -86,7 +82,8 @@ azureCreateBatchAccount <- function(azureActiveContext, batchAccount, TRUE } -#' Delete an Azure Batch Account. + +#' Delete an azure batch account. #' #' @inheritParams setAzureContext #' @inheritParams azureAuthenticate @@ -97,8 +94,6 @@ azureCreateBatchAccount <- function(azureActiveContext, batchAccount, azureDeleteBatchAccount <- function(azureActiveContext, batchAccount, resourceGroup, subscriptionID, verbose = FALSE) { assert_that(is.azureActiveContext(azureActiveContext)) - azureCheckToken(azureActiveContext) - azToken <- azureActiveContext$Token if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID @@ -125,6 +120,7 @@ azureDeleteBatchAccount <- function(azureActiveContext, batchAccount, TRUE } + #' Get the Batch Keys for Specified Batch Account. #' #' @inheritParams setAzureContext @@ -135,8 +131,6 @@ azureDeleteBatchAccount <- function(azureActiveContext, batchAccount, azureBatchGetKey <- function(azureActiveContext, batchAccount, resourceGroup, subscriptionID, verbose = FALSE) { assert_that(is.azureActiveContext(azureActiveContext)) - azureCheckToken(azureActiveContext) - azToken <- azureActiveContext$Token if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID diff --git a/R/AzureSMR-package.R b/R/AzureSMR-package.R index 81111a0..489dcb1 100644 --- a/R/AzureSMR-package.R +++ b/R/AzureSMR-package.R @@ -28,6 +28,11 @@ #' - [azureDeleteHDI()] #' - [azureRunScriptAction()] #' - [azureScriptActionHistory()] +#' * Azure batch: +#' - [azureListBatchAccounts()] +#' - [azureCreateBatchAccount()] +#' - [azureDeleteBatchAccount()] +#' - [azureBatchGetKey()] #' #' #' @name AzureSMR diff --git a/man/AzureSMR.Rd b/man/AzureSMR.Rd index 8e4a5f9..2c8a0bc 100644 --- a/man/AzureSMR.Rd +++ b/man/AzureSMR.Rd @@ -46,6 +46,13 @@ This enables you to use and change many Azure resources. The following is an inc \item \code{\link[=azureRunScriptAction]{azureRunScriptAction()}} \item \code{\link[=azureScriptActionHistory]{azureScriptActionHistory()}} } +\item Azure batch: +\itemize{ +\item \code{\link[=azureListBatchAccounts]{azureListBatchAccounts()}} +\item \code{\link[=azureCreateBatchAccount]{azureCreateBatchAccount()}} +\item \code{\link[=azureDeleteBatchAccount]{azureDeleteBatchAccount()}} +\item \code{\link[=azureBatchGetKey]{azureBatchGetKey()}} +} } } \keyword{package} diff --git a/man/azureBatchGetKey.Rd b/man/azureBatchGetKey.Rd index 480eb52..e8fc7d5 100644 --- a/man/azureBatchGetKey.Rd +++ b/man/azureBatchGetKey.Rd @@ -8,11 +8,11 @@ azureBatchGetKey(azureActiveContext, batchAccount, resourceGroup, subscriptionID, verbose = FALSE) } \arguments{ -\item{azureActiveContext}{A container used for caching variables used by `AzureSMR`} +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} \item{resourceGroup}{Name of the resource group} -\item{subscriptionID}{Subscription ID. This is obtained automatically by [azureAuthenticate()] when only a single subscriptionID is available via Active Directory} +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} \item{verbose}{Print Tracing information (Default False)} } @@ -24,4 +24,3 @@ Other Batch account functions: \code{\link{azureCreateBatchAccount}}, \code{\link{azureDeleteBatchAccount}}, \code{\link{azureListBatchAccounts}} } - diff --git a/man/azureCreateBatchAccount.Rd b/man/azureCreateBatchAccount.Rd index ae6cc53..c027f3d 100644 --- a/man/azureCreateBatchAccount.Rd +++ b/man/azureCreateBatchAccount.Rd @@ -2,31 +2,30 @@ % Please edit documentation in R/AzureBatch.R \name{azureCreateBatchAccount} \alias{azureCreateBatchAccount} -\title{Create an Azure Batch Account.} +\title{Create an azure batch account.} \usage{ azureCreateBatchAccount(azureActiveContext, batchAccount, location = "northeurope", resourceGroup, subscriptionID, asynchronous = FALSE, verbose = FALSE) } \arguments{ -\item{azureActiveContext}{A container used for caching variables used by `AzureSMR`} +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} \item{location}{A string for the location to create batch account} \item{resourceGroup}{Name of the resource group} -\item{subscriptionID}{Subscription ID. This is obtained automatically by [azureAuthenticate()] when only a single subscriptionID is available via Active Directory} +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} \item{asynchronous}{If TRUE, submits asynchronous request to Azure. Otherwise waits until batch account is created.} \item{verbose}{Print Tracing information (Default False)} } \description{ -Create an Azure Batch Account. +Create an azure batch account. } \seealso{ Other Batch account functions: \code{\link{azureBatchGetKey}}, \code{\link{azureDeleteBatchAccount}}, \code{\link{azureListBatchAccounts}} } - diff --git a/man/azureDeleteBatchAccount.Rd b/man/azureDeleteBatchAccount.Rd index 45ce7fd..5ebcb17 100644 --- a/man/azureDeleteBatchAccount.Rd +++ b/man/azureDeleteBatchAccount.Rd @@ -2,26 +2,25 @@ % Please edit documentation in R/AzureBatch.R \name{azureDeleteBatchAccount} \alias{azureDeleteBatchAccount} -\title{Delete an Azure Batch Account.} +\title{Delete an azure batch account.} \usage{ azureDeleteBatchAccount(azureActiveContext, batchAccount, resourceGroup, subscriptionID, verbose = FALSE) } \arguments{ -\item{azureActiveContext}{A container used for caching variables used by `AzureSMR`} +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} \item{resourceGroup}{Name of the resource group} -\item{subscriptionID}{Subscription ID. This is obtained automatically by [azureAuthenticate()] when only a single subscriptionID is available via Active Directory} +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} \item{verbose}{Print Tracing information (Default False)} } \description{ -Delete an Azure Batch Account. +Delete an azure batch account. } \seealso{ Other Batch account functions: \code{\link{azureBatchGetKey}}, \code{\link{azureCreateBatchAccount}}, \code{\link{azureListBatchAccounts}} } - diff --git a/man/azureListBatchAccounts.Rd b/man/azureListBatchAccounts.Rd index 7943299..181f31f 100644 --- a/man/azureListBatchAccounts.Rd +++ b/man/azureListBatchAccounts.Rd @@ -8,11 +8,11 @@ azureListBatchAccounts(azureActiveContext, resourceGroup, subscriptionID, verbose = FALSE) } \arguments{ -\item{azureActiveContext}{A container used for caching variables used by `AzureSMR`} +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} \item{resourceGroup}{Name of the resource group} -\item{subscriptionID}{Subscription ID. This is obtained automatically by [azureAuthenticate()] when only a single subscriptionID is available via Active Directory} +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} \item{verbose}{Print Tracing information (Default False)} } @@ -24,4 +24,3 @@ Other Batch account functions: \code{\link{azureBatchGetKey}}, \code{\link{azureCreateBatchAccount}}, \code{\link{azureDeleteBatchAccount}} } - From f8f44ff6e7fc1e3cd8b32f4b6f43002033953d88 Mon Sep 17 00:00:00 2001 From: Brian Date: Wed, 14 Jun 2017 14:06:00 -0700 Subject: [PATCH 13/20] Added batch test --- tests/testthat/test-batch.R | 82 +++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tests/testthat/test-batch.R diff --git a/tests/testthat/test-batch.R b/tests/testthat/test-batch.R new file mode 100644 index 0000000..44d4513 --- /dev/null +++ b/tests/testthat/test-batch.R @@ -0,0 +1,82 @@ +if (interactive()) library("testthat") + +settingsfile <- system.file("tests/testthat/config.json", package = "AzureSMR") +config <- read.AzureSMR.config(settingsfile) + +# ------------------------------------------------------------------------ + +context("Batch") + +asc <- createAzureContext() +with(config, + setAzureContext(asc, tenantID = tenantID, clientID = clientID, authKey = authKey) +) + +azureAuthenticate(asc) + +timestamp <- format(Sys.time(), format = "%y%m%d%H%M") +resourceGroup_name <- paste0("AzureSMtest_", timestamp) +batch_account <- paste0("azuresmr", timestamp) +batch_location = "westeurope" + +test_that("Can create resource group", { + skip_if_missing_config(settingsfile) + + res <- azureCreateResourceGroup(asc, location = "westeurope", resourceGroup = resourceGroup_name) + expect_equal(res, TRUE) + + wait_for_azure( + resourceGroup_name %in% azureListRG(asc)$resourceGroup + ) + expect_true(resourceGroup_name %in% azureListRG(asc)$resourceGroup) +}) + + +context(" - batch account") +test_that("create batch account", { + skip_if_missing_config(settingsfile) + + res <- azureCreateBatchAccount(asc, + batchAccount = batch_account, + resourceGroup = resourceGroup_name, + location = batch_location) + + if(res == "Account already exists with the same name") skip("Account already exists with the same name") + expect_equal(res, TRUE) + + wait_for_azure( + batch_account %in% azureListBatchAccounts(asc)$name + ) + + expect_true(batch_account %in% azureListBatchAccounts(asc)$name) +}) + +context(" - batch account list keys") +test_that("list keys", { + skip_if_missing_config(settingsfile) + + wait_for_azure( + batch_account %in% azureListBatchAccounts(asc)$name + ) + + res <- azureBatchGetKey(asc, + batchAccount = batch_account, + resourceGroup = resourceGroup_name) + + expect_true(is_storage_key(res)) +}) + +context(" - delete batch account") +test_that("can delete batch account", { + skip_if_missing_config(settingsfile) + + # delete the actual batch account + expect_true( + azureDeleteBatchAccount(asc, + batchAccount = batch_account, + resourceGroup = resourceGroup_name, + subscriptionID = asc$subscriptionID) + ) + + azureDeleteResourceGroup(asc, resourceGroup = resourceGroup_name) +}) \ No newline at end of file From c7f229e55f6e64d358fec12da2b3274eb0dd121a Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Thu, 15 Jun 2017 08:56:14 +0800 Subject: [PATCH 14/20] Rewrote roxygen. Removed @title, added @note, and replaced @param with @inheritParams. --- R/AzureCost.R | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/R/AzureCost.R b/R/AzureCost.R index 021e449..ac29e00 100644 --- a/R/AzureCost.R +++ b/R/AzureCost.R @@ -1,8 +1,8 @@ -#' @title Get data consumption of an Azure subscription for a time period. Aggregation method can be either daily based or hourly based. +#' Get data consumption of an Azure subscription for a time period. Aggregation method can be either daily based or hourly based. #' #' @note Formats of start time point and end time point follow ISO 8601 standard. Say if one would like to calculate data consumption between Feb 21, 2017 to Feb 25, 2017, with an aggregation granularity of "daily based", the inputs should be "2017-02-21 00:00:00" and "2017-02-25 00:00:00", for start time point and end time point, respectively. If the aggregation granularity is hourly based, the inputs can be "2017-02-21 01:00:00" and "2017-02-21 02:00:00", for start and end time point, respectively. NOTE by default the Azure data consumption API does not allow an aggregation granularity that is finer than an hour. In the case of "hourly based" granularity, if the time difference between start and end time point is less than an hour, data consumption will still be calculated hourly based with end time postponed. For example, if the start time point and end time point are "2017-02-21 00:00:00" and "2017-02-21 00:45:00", the actual returned results are are data consumption in the interval of "2017-02-21 00:00:00" and "2017-02-21 01:00:00". However this calculation is merely for retrieving the information of an existing DSVM instance (e.g., meterId) with which the pricing rate is multiplied by to obtain the overall expense. Time zone of all time inputs are synchronized to UTC. #' -#' @param azureActiveContext AzureSMR context object. +#' @inheritParams setAzureContext #' #' @param instance Instance of Azure DSVM name that one would like to check expense. It is by default empty, which returns data consumption for all instances under subscription. #' @@ -201,9 +201,9 @@ azureDataConsumption <- function(azureActiveContext, } } -#' @title Get pricing details of resources under a subscription. +#' Get pricing details of resources under a subscription. #' -#' @param azureActiveContext - Azure Context Object. +#' @inheritParams setAzureContext #' #' @param currency Currency in which price rating is measured. #' @@ -213,6 +213,8 @@ azureDataConsumption <- function(azureActiveContext, #' #' @param region region information about the subscription. #' +#' @note The pricing rates function wraps API calls to Azure RateCard and current only the API supports only for Pay-As-You-Go offer scheme. +#' #' @export azurePricingRates <- function(azureActiveContext, currency, @@ -272,9 +274,9 @@ azurePricingRates <- function(azureActiveContext, return(df_meter) } -#' @title Calculate cost of using a specific instance of Azure for certain period. +#' Calculate cost of using a specific instance of Azure for certain period. #' -#' @param azureActiveContext AzureSMR context. +#' @inheritParams setAzureContext #' #' @param instance Instance of Azure instance that one would like to check expense. No matter whether resource group is given or not, if a instance of instance is given, data consumption of that instance is returned. #' From 91bc21aa136c5f2f46f28e1faf595e55133cc724 Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Thu, 15 Jun 2017 09:22:09 +0800 Subject: [PATCH 15/20] Lower case the upper case object names --- R/AzureCost.R | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/R/AzureCost.R b/R/AzureCost.R index ac29e00..e412e76 100644 --- a/R/AzureCost.R +++ b/R/AzureCost.R @@ -80,30 +80,30 @@ azureDataConsumption <- function(azureActiveContext, # reformat time variables to make them compatible with API call. - START <- URLencode(paste(as.Date(timeStart), "T", + start <- URLencode(paste(as.Date(timeStart), "T", sprintf("%02d", lubridate::hour(timeStart)), ":", sprintf("%02d", lubridate::minute(timeStart)), ":", sprintf("%02d", lubridate::second(timeStart)), "+", "00:00", sep=""), reserved=TRUE) - END <- URLencode(paste(as.Date(timeEnd), "T", + end <- URLencode(paste(as.Date(timeEnd), "T", sprintf("%02d", lubridate::hour(timeEnd)), ":", sprintf("%02d", lubridate::minute(timeEnd)), ":", sprintf("%02d", lubridate::second(timeEnd)), "+", "00:00", sep=""), reserved=TRUE) - URL <- + url <- sprintf("https://management.azure.com/subscriptions/%s/providers/Microsoft.Commerce/UsageAggregates?api-version=%s&reportedStartTime=%s&reportedEndTime=%s&aggregationgranularity=%s&showDetails=%s", azureActiveContext$subscriptionID, "2015-06-01-preview", - START, - END, + start, + end, granularity, "false" ) r <- call_azure_sm(azureActiveContext, - uri=URL, + uri=url, verb="GET", verbose=verbose) @@ -175,8 +175,8 @@ azureDataConsumption <- function(azureActiveContext, # NOTE the maximum number of records returned from API is limited to 1000. - if (nrow(df_use) == 1000 && max(as.POSIXct(df_use$usageEndTime)) < as.POSIXct(END)) { - warning(sprintf("The number of records in the specified time period %s to %s exceeds the limit that can be returned from API call. Consumption information is truncated. Please use a small period instead.", START, END)) + if (nrow(df_use) == 1000 && max(as.POSIXct(df_use$usageEndTime)) < as.POSIXct(end)) { + warning(sprintf("The number of records in the specified time period %s to %s exceeds the limit that can be returned from API call. Consumption information is truncated. Please use a small period instead.", start, end)) } df_use %<>% From 1133535b19937b94d0d9fb8b00bab0a30dd60604 Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Thu, 15 Jun 2017 11:26:16 +0800 Subject: [PATCH 16/20] Remove use of dplyr and magrittr --- R/AzureCost.R | 140 ++++++++++++++++++++++---------------------------- 1 file changed, 61 insertions(+), 79 deletions(-) diff --git a/R/AzureCost.R b/R/AzureCost.R index e412e76..7c2eaec 100644 --- a/R/AzureCost.R +++ b/R/AzureCost.R @@ -113,13 +113,9 @@ azureDataConsumption <- function(azureActiveContext, df <- fromJSON(rl) - df_use <- - df$value$properties %>% - select(-infoFields) + df_use <- df$value$properties - inst_data <- - df$value$properties$instanceData %>% - lapply(., fromJSON) + inst_data <- lapply(df$value$properties$instanceData, fromJSON) # retrieve results that match instance name. @@ -149,27 +145,10 @@ azureDataConsumption <- function(azureActiveContext, time_diff <- as.numeric(de - ds) / 3600 - df_use %<>% - select(usageStartTime, - usageEndTime, - meterName, - meterCategory, - meterSubCategory, - unit, - meterId, - quantity, - meterRegion) %>% - filter(meterName == "Compute Hours") %>% - filter(row_number() == 1) %>% - mutate(quantity = time_diff) %>% - mutate(usageStartTime = as.POSIXct(usageStartTime)) %>% - mutate(usageEndTime = as.POSIXct(usageEndTime)) + df_use <- df_use[which(df_use$meterName == "Compute Hours"), ] + df_use <- df_use[1, ] - writeLines(sprintf("The data consumption for %s between %s and %s is", - instance, - as.character(timeStart), - as.character(timeEnd))) - return(df_use) + df_use$quantity <- df_use$time_diff } else { @@ -178,27 +157,27 @@ azureDataConsumption <- function(azureActiveContext, if (nrow(df_use) == 1000 && max(as.POSIXct(df_use$usageEndTime)) < as.POSIXct(end)) { warning(sprintf("The number of records in the specified time period %s to %s exceeds the limit that can be returned from API call. Consumption information is truncated. Please use a small period instead.", start, end)) } - - df_use %<>% - select(usageStartTime, - usageEndTime, - meterName, - meterCategory, - meterSubCategory, - unit, - meterId, - quantity, - meterRegion) %>% - mutate(usageStartTime = as.POSIXct(usageStartTime)) %>% - mutate(usageEndTime = as.POSIXct(usageEndTime)) - - writeLines(sprintf("The data consumption for %s between %s and %s is", - instance, - as.character(timeStart), - as.character(timeEnd))) - - df_use } + + df_use <- df_use[, c("usageStartTime", + "usageEndTime", + "meterName", + "meterCategory", + "meterSubCategory", + "unit", + "meterId", + "quantity", + "meterRegion")] + + df_use$usageStartTime <- as.POSIXct(df_use$usageStartTime) + df_use$usageEndTime <- as.POSIXct(df_use$usageEndTime) + + writeLines(sprintf("The data consumption for %s between %s and %s is", + instance, + as.character(timeStart), + as.character(timeEnd))) + + return(df_use) } #' Get pricing details of resources under a subscription. @@ -259,8 +238,6 @@ azurePricingRates <- function(azureActiveContext, stopWithAzureError(r) - # r <- GET(url, add_headers(.headers=c(Authorization=azureActiveContext$Token, "Content-Type"="application/json"))) - rl <- fromJSON(content(r, "text", encoding="UTF-8"), simplifyDataFrame=TRUE) df_meter <- rl$Meters @@ -271,7 +248,9 @@ azurePricingRates <- function(azureActiveContext, df_meter <- subset(df_meter, select=-MeterRates) df_meter <- subset(df_meter, select=-MeterTags) - return(df_meter) + names(df_meter) <- paste0(tolower(substring(names(df_meter), 1, 1)), substring(names(df_meter), 2)) + + df_meter } #' Calculate cost of using a specific instance of Azure for certain period. @@ -309,26 +288,18 @@ azureExpenseCalculator <- function(azureActiveContext, offerId, region, verbose=FALSE) { - df_use <- - azureDataConsumption(azureActiveContext, - instance=instance, - timeStart=timeStart, - timeEnd=timeEnd, - granularity=granularity, - verbose=verbose) %>% - select(meterId, - meterSubCategory, - usageStartTime, - usageEndTime, - quantity) - - df_used_data <- - group_by(df_use, meterId) %>% - arrange(usageStartTime, usageEndTime) %>% - summarise(usageStartDate=as.Date(min(usageStartTime), tz=Sys.timezone()), - usageEndDate=as.Date(max(usageEndTime), tz=Sys.timezone()), - totalQuantity=sum(quantity)) %>% - ungroup() + df_use <- azureDataConsumption(azureActiveContext, + instance=instance, + timeStart=timeStart, + timeEnd=timeEnd, + granularity=granularity, + verbose=verbose) + + df_used_data <- df_use[, c("meterId", + "meterSubCategory", + "usageStartTime", + "usageEndTime", + "quantity")] # use meterId to find pricing rates and then calculate total cost. @@ -339,19 +310,30 @@ azureExpenseCalculator <- function(azureActiveContext, offerId=offerId, verbose=verbose) - meter_list <- df_used_data$meterId + meter_list <- unique(df_used_data$meterId) + + df_used_rates <- df_rates[which(df_rates$meterId %in% meter_list), ] + df_used_rates$meterId <- df_used_rates$meterId + + # join data consumption and meter pricing rate. + + df_merged <- merge(x=df_used_data, + y=df_used_rates, + by="meterId", + all.x=TRUE) - df_used_rates <- - filter(df_rates, MeterId %in% meter_list) %>% - rename(meterId=MeterId) + df_merged$meterSubCategory <- df_merged$meterSubCategory.y + df_merged$cost <- df_merged$quantity * df_merged$meterRate - df_cost <- - left_join(df_used_data, df_used_rates, by="meterId") %>% - mutate(Cost=totalQuantity * MeterRate) %>% - select(-IncludedQuantity, -EffectiveDate, -MeterStatus, -usageStartDate, -usageEndDate, -meterId, -MeterRegion) %>% - na.omit() + df_cost <- df_merged[, c("meterName", + "meterCategory", + "meterSubCategory", + "quantity", + "unit", + "meterRate", + "cost")] - # reorder columns. + names(df_cost) <- paste0(tolower(substring(names(df_cost), 1, 1)), substring(names(df_cost), 2)) df_cost <- df_cost[, c(3, 2, 4, 1, 5, 6, 7)] From 82e6cd3dc16bbce7771e9ab633ffabb3b96652ee Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Thu, 15 Jun 2017 11:28:31 +0800 Subject: [PATCH 17/20] Rewrote testing conditions --- tests/testthat/test-cost.R | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/testthat/test-cost.R b/tests/testthat/test-cost.R index a308ea5..643e7b5 100644 --- a/tests/testthat/test-cost.R +++ b/tests/testthat/test-cost.R @@ -25,9 +25,9 @@ sa_name <- paste0("azuresmr", timestamp) # run test. -# get cost by day. +# get data consumption by day. -test_that("Get cost by day", { +test_that("Get data consumption by day", { skip_if_missing_config(settingsfile) time_end <- paste0(as.Date(Sys.Date()), "00:00:00") @@ -38,7 +38,7 @@ test_that("Get cost by day", { timeEnd=time_end, granularity="Daily") - expect_type(res, type="list") + expect_is(res, class="data.frame") expect_identical(object=names(res), expected=c("usageStartTime", "usageEndTime", "meterName", @@ -61,17 +61,17 @@ test_that("Get pricing rates", { offerId=config$OFFER, region=config$REGION) - expect_type(res, type="list") - expect_identical(object=names(res), expected=c("EffectiveDate", - "IncludedQuantity", - "MeterCategory", - "MeterId", - "MeterName", - "MeterRegion", - "MeterStatus", - "MeterSubCategory", - "Unit", - "MeterRate")) + expect_is(res, class="data.frame") + expect_identical(object=names(res), expected=c("effectiveDate", + "includedQuantity", + "meterCategory", + "meterId", + "meterName", + "meterRegion", + "meterStatus", + "meterSubCategory", + "unit", + "meterRate")) }) @@ -92,12 +92,12 @@ test_that("Get cost by day", { offerId=config$OFFER, region=config$REGION) - expect_type(res, type="list") - expect_identical(object=names(res), expected=c("MeterName", - "MeterCategory", - "MeterSubCategory", - "totalQuantity", - "Unit", - "MeterRate", - "Cost")) + expect_is(res, class="data.frame") + expect_identical(object=names(res), expected=c("meterName", + "meterCategory", + "meterSubCategory", + "quantity", + "unit", + "meterRate", + "cost")) }) \ No newline at end of file From e9c8a98c50980784d13cd6e739c699a2275b2644 Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Thu, 15 Jun 2017 11:31:22 +0800 Subject: [PATCH 18/20] No need to reorder --- R/AzureCost.R | 2 -- 1 file changed, 2 deletions(-) diff --git a/R/AzureCost.R b/R/AzureCost.R index 7c2eaec..1d82524 100644 --- a/R/AzureCost.R +++ b/R/AzureCost.R @@ -335,7 +335,5 @@ azureExpenseCalculator <- function(azureActiveContext, names(df_cost) <- paste0(tolower(substring(names(df_cost), 1, 1)), substring(names(df_cost), 2)) - df_cost <- df_cost[, c(3, 2, 4, 1, 5, 6, 7)] - df_cost } From 27f0afb17df98f9b34a6579914cb5557a8b5421a Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Thu, 15 Jun 2017 11:49:55 +0800 Subject: [PATCH 19/20] Wrap to 80 --- R/AzureCost.R | 146 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 105 insertions(+), 41 deletions(-) diff --git a/R/AzureCost.R b/R/AzureCost.R index 1d82524..c5e7582 100644 --- a/R/AzureCost.R +++ b/R/AzureCost.R @@ -1,16 +1,38 @@ -#' Get data consumption of an Azure subscription for a time period. Aggregation method can be either daily based or hourly based. +#' Get data consumption of an Azure subscription for a time period. Aggregation +#' method can be either daily based or hourly based. #' -#' @note Formats of start time point and end time point follow ISO 8601 standard. Say if one would like to calculate data consumption between Feb 21, 2017 to Feb 25, 2017, with an aggregation granularity of "daily based", the inputs should be "2017-02-21 00:00:00" and "2017-02-25 00:00:00", for start time point and end time point, respectively. If the aggregation granularity is hourly based, the inputs can be "2017-02-21 01:00:00" and "2017-02-21 02:00:00", for start and end time point, respectively. NOTE by default the Azure data consumption API does not allow an aggregation granularity that is finer than an hour. In the case of "hourly based" granularity, if the time difference between start and end time point is less than an hour, data consumption will still be calculated hourly based with end time postponed. For example, if the start time point and end time point are "2017-02-21 00:00:00" and "2017-02-21 00:45:00", the actual returned results are are data consumption in the interval of "2017-02-21 00:00:00" and "2017-02-21 01:00:00". However this calculation is merely for retrieving the information of an existing DSVM instance (e.g., meterId) with which the pricing rate is multiplied by to obtain the overall expense. Time zone of all time inputs are synchronized to UTC. +#' @note Formats of start time point and end time point follow ISO 8601 standard +#' Say if one would like to calculate data consumption between Feb 21, 2017 to +#' Feb 25, 2017, with an aggregation granularity of "daily based", the inputs +#' should be "2017-02-21 00:00:00" and "2017-02-25 00:00:00", for start time +#' point and end time point, respectively. If the aggregation granularity is +#' hourly based, the inputs can be "2017-02-21 01:00:00" and +#' "2017-02-21 02:00:00", for start and end time point, respectively. +#' NOTE by default the Azure data +#' consumption API does not allow an aggregation granularity that is finer +#' than an hour. In the case of "hourly based" granularity, if the time +#' difference between start and end time point is less than an hour, data +#' consumption will still be calculated hourly based with end time postponed. +#' For example, if the start time point and end time point are "2017-02-21 +#' 00:00:00" and "2017-02-21 00:45:00", the actual returned results are +#' data consumption in the interval of "2017-02-21 00:00:00" and +#' "2017-02-21 01:00:00". However this calculation is merely for retrieving +#' the information of an existing instance instance (e.g., meterId) with +#' which the pricing rate is multiplied by to obtain the overall expense. +#' Time zone of all time inputs are synchronized to UTC. #' #' @inheritParams setAzureContext #' -#' @param instance Instance of Azure DSVM name that one would like to check expense. It is by default empty, which returns data consumption for all instances under subscription. +#' @param instance Instance name that one would like to check expe +#' nse. It is by default empty, which returns data consumption for +#' all instances under subscription. #' #' @param timeStart Start time. #' #' @param timeEnd End time. #' -#' @param granularity Aggregation granularity. Can be either "Daily" or "Hourly". +#' @param granularity Aggregation granularity. Can be either "Daily" or +#' "Hourly". #' @export azureDataConsumption <- function(azureActiveContext, instance="", @@ -64,14 +86,29 @@ azureDataConsumption <- function(azureActiveContext, } - # If the computation time is less than a hour, timeEnd will be incremented by an hour to get the total cost within an hour aggregated from timeStart. However, only the consumption on computation is considered in the returned data, and the computation consumption will then be replaced with the actual timeEnd - timeStart. - - # NOTE: estimation of cost in this case is rough though, it captures the major component of total cost, which originates from running an Azure instance. Other than computation cost, there are also cost on activities such as data transfer, software library license, etc. This is not included in the approximation here until a solid method for capturing those consumption data is found. Data ingress does not generate cost, but data egress does. Usually the occurrence of data transfer is not that frequent as computation, and pricing rates for data transfer is also less than computation (e.g., price rate of "data transfer in" is ~ 40% of that of computation on an A3 virtual machine). + # If the computation time is less than a hour, timeEnd will be incremented by + # an hour to get the total cost within an hour aggregated from timeStart. + # However, only the consumption on computation is considered in the returned + # data, and the computation consumption will then be replaced with the actual + # timeEnd - timeStart. + + # NOTE: estimation of cost in this case is rough though, it captures the major + # component of total cost, which originates from running an Azure instance. + # Other than computation cost, there are also cost on activities such as data + # transfer, software library license, etc. This is not included in the + # approximation here until a solid method for capturing those consumption data + # is found. Data ingress does not generate cost, but data egress does. Usually + # the occurrence of data transfer is not that frequent as computation, and + # pricing rates for data transfer is also less than computation (e.g., price + # rate of "data transfer in" is ~ 40% of that of computation on an A3 virtual + # machine). # TODO: inlude other types of cost for jobs that take less than an hour. if (as.numeric(timeEnd - timeStart) == 0) { - writeLines("Difference between timeStart and timeEnd is less than the aggregation granularity. Cost is estimated solely on computation running time.") + writeLines("Difference between timeStart and timeEnd is less than the + aggregation granularity. Cost is estimated solely on computation + running time.") # increment timeEnd by one hour. @@ -80,20 +117,35 @@ azureDataConsumption <- function(azureActiveContext, # reformat time variables to make them compatible with API call. - start <- URLencode(paste(as.Date(timeStart), "T", - sprintf("%02d", lubridate::hour(timeStart)), ":", sprintf("%02d", lubridate::minute(timeStart)), ":", sprintf("%02d", lubridate::second(timeStart)), "+", + start <- URLencode(paste(as.Date(timeStart), + "T", + sprintf("%02d", lubridate::hour(timeStart)), + ":", + sprintf("%02d", lubridate::minute(timeStart)), + ":", + sprintf("%02d", lubridate::second(timeStart)), + "+", "00:00", sep=""), reserved=TRUE) - end <- URLencode(paste(as.Date(timeEnd), "T", - sprintf("%02d", lubridate::hour(timeEnd)), ":", sprintf("%02d", lubridate::minute(timeEnd)), ":", sprintf("%02d", lubridate::second(timeEnd)), "+", + end <- URLencode(paste(as.Date(timeEnd), + "T", + sprintf("%02d", lubridate::hour(timeEnd)), + ":", + sprintf("%02d", lubridate::minute(timeEnd)), + ":", + sprintf("%02d", lubridate::second(timeEnd)), + "+", "00:00", sep=""), reserved=TRUE) url <- - sprintf("https://management.azure.com/subscriptions/%s/providers/Microsoft.Commerce/UsageAggregates?api-version=%s&reportedStartTime=%s&reportedEndTime=%s&aggregationgranularity=%s&showDetails=%s", + sprintf("https://management.azure.com/subscriptions/%s/providers/ + Microsoft.Commerce/UsageAggregates?api-version=%s + &reportedStartTime=%s&reportedEndTime=%s + &aggregationgranularity=%s&showDetails=%s", azureActiveContext$subscriptionID, "2015-06-01-preview", start, @@ -128,16 +180,19 @@ azureDataConsumption <- function(azureActiveContext, if(!missing(instance)) { if(length(index_instance) == 0) - stop("No data consumption records found for the instance during the given period.") + stop("No data consumption records found for the instance during the + given period.") df_use <- df_use[index_instance, ] } else if(missing(instance)) { if(length(index_resource) == 0) - stop("No data consumption records found for the resource group during the given period.") + stop("No data consumption records found for the resource group during + the given period.") df_use <- df_use[index_resource, ] } } - # if time difference is less than one hour. Only return one row of computation consumption whose value is the time difference. + # if time difference is less than one hour. Only return one row of computation + # consumption whose value is the time difference. # timeEnd <- timeEnd - 3600 @@ -154,8 +209,12 @@ azureDataConsumption <- function(azureActiveContext, # NOTE the maximum number of records returned from API is limited to 1000. - if (nrow(df_use) == 1000 && max(as.POSIXct(df_use$usageEndTime)) < as.POSIXct(end)) { - warning(sprintf("The number of records in the specified time period %s to %s exceeds the limit that can be returned from API call. Consumption information is truncated. Please use a small period instead.", start, end)) + if (nrow(df_use) == 1000 && + max(as.POSIXct(df_use$usageEndTime)) < as.POSIXct(end)) { + warning(sprintf("The number of records in the specified time period %s + to %s exceeds the limit that can be returned from API call. + Consumption information is truncated. Please use a small + period instead.", timeStart, timeEnd)) } } @@ -188,11 +247,13 @@ azureDataConsumption <- function(azureActiveContext, #' #' @param locale Locality information of subscription. #' -#' @param offerId Offer ID of the subscription. Detailed information can be found at https://azure.microsoft.com/en-us/support/legal/offer-details/ +#' @param offerId Offer ID of the subscription. Detailed information can be +#' found at https://azure.microsoft.com/en-us/support/legal/offer-details/ #' #' @param region region information about the subscription. #' -#' @note The pricing rates function wraps API calls to Azure RateCard and current only the API supports only for Pay-As-You-Go offer scheme. +#' @note The pricing rates function wraps API calls to Azure RateCard and +#' current only the API supports only for Pay-As-You-Go offer scheme. #' #' @export azurePricingRates <- function(azureActiveContext, @@ -221,8 +282,10 @@ azurePricingRates <- function(azureActiveContext, stop("Error: please provide region information.") url <- paste( - "https://management.azure.com/subscriptions/", azureActiveContext$subscriptionID, - "/providers/Microsoft.Commerce/RateCard?api-version=2016-08-31-preview&$filter=", + "https://management.azure.com/subscriptions/", + azureActiveContext$subscriptionID, + "/providers/Microsoft.Commerce/RateCard?api-version=2016-08-31-preview& + $filter=", "OfferDurableId eq '", offerId, "'", " and Currency eq '", currency, "'", " and Locale eq '", locale, "'", @@ -243,12 +306,16 @@ azurePricingRates <- function(azureActiveContext, df_meter <- rl$Meters df_meter$MeterRate <- rl$Meters$MeterRates$`0` - # an irresponsible drop of MeterRates and MeterTags. Will add them back after having a better handle of them. + # NOTE: an irresponsible drop of MeterRates and MeterTags. Will add them back + # after having a better handle of them. df_meter <- subset(df_meter, select=-MeterRates) df_meter <- subset(df_meter, select=-MeterTags) - names(df_meter) <- paste0(tolower(substring(names(df_meter), 1, 1)), substring(names(df_meter), 2)) + names(df_meter) <- paste0(tolower(substring(names(df_meter), + 1, + 1)), + substring(names(df_meter), 2)) df_meter } @@ -257,25 +324,19 @@ azurePricingRates <- function(azureActiveContext, #' #' @inheritParams setAzureContext #' -#' @param instance Instance of Azure instance that one would like to check expense. No matter whether resource group is given or not, if a instance of instance is given, data consumption of that instance is returned. -#' -#' @param timeStart Start time. -#' -#' @param timeEnd End time. -#' -#' @param granularity Aggregation granularity. Can be either "Daily" or "Hourly". +#' @inheritParams azureDataConsumption #' -#' @param currency Currency in which price rating is measured. -#' -#' @param locale Locality information of subscription. -#' -#' @param offerId Offer ID of the subscription. Detailed information can be found at https://azure.microsoft.com/en-us/support/legal/offer-details/ -#' -#' @param region region information about the subscription. +#' @inheritParams azurePricingRates #' -#' @return Total cost measured in the given currency of the specified Azure instance in the period. +#' @return Total cost measured in the given currency of the specified Azure +#' instance in the period. #' -#' @note Note if difference between \code{timeStart} and \code{timeEnd} is less than the finest granularity, e.g., "Hourly" (we notice this is a usual case when one needs to be aware of the charges of a job that takes less than an hour), the expense will be estimated based solely on computation hour. That is, the total expense is the multiplication of computation hour and pricing rate of the DSVM instance. +#' @note Note if difference between \code{timeStart} and \code{timeEnd} is +#' less than the finest granularity, e.g., "Hourly" (we notice this is a +#' usual case when one needs to be aware of the charges of a job that takes +#' less than an hour), the expense will be estimated based solely on computation +#' hour. That is, the total expense is the multiplication of computation hour +#' and pricing rate of the requested instance. #' #' @export azureExpenseCalculator <- function(azureActiveContext, @@ -333,7 +394,10 @@ azureExpenseCalculator <- function(azureActiveContext, "meterRate", "cost")] - names(df_cost) <- paste0(tolower(substring(names(df_cost), 1, 1)), substring(names(df_cost), 2)) + names(df_cost) <- paste0(tolower(substring(names(df_cost), + 1, + 1)), + substring(names(df_cost), 2)) df_cost } From cbf05cdf553e90b045d66527e1246f4a636fb420 Mon Sep 17 00:00:00 2001 From: yueguoguo Date: Thu, 15 Jun 2017 11:55:43 +0800 Subject: [PATCH 20/20] Remove dependencies on dplyr and magrittr --- DESCRIPTION | 17 +++++++++-------- R/AzureSMR-package.R | 3 --- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index ef0c2d2..98a5432 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,8 +2,8 @@ Package: AzureSMR Title: Manage and Interact with Azure Resources Description: Helps users to manage Azure Services and objects from within an R Session. This includes Azure Storage (e.g. containers and blobs), Virtual - Machines and HDInsight (Spark, Hive). To use the package, you must configure - an Azure Active Directory application and service principal in the Azure portal. + Machines and HDInsight (Spark, Hive). To use the package, you must configure an + Azure Active Directory application and service principal in the Azure portal. Type: Package Version: 0.2.5 Date: 2017-06-06 @@ -18,16 +18,17 @@ URL: https://github.com/Microsoft/AzureSMR BugReports: https://github.com/Microsoft/AzureSMR/issues NeedsCompilation: no Imports: - assertthat, + assertthat, httr, jsonlite, XML, base64enc, digest, - shiny (>= 0.13), - miniUI (>= 0.1.1), - rstudioapi (>= 0.5), - DT + shiny (>= 0.13), + miniUI (>= 0.1.1), + rstudioapi (>= 0.5), + DT, + lubridate, Depends: R(>= 3.0.0) Suggests: @@ -36,5 +37,5 @@ Suggests: testthat VignetteBuilder: knitr LazyData: TRUE -RoxygenNote: 6.0.1 +RoxygenNote: 5.0.1 Roxygen: list(markdown = TRUE) diff --git a/R/AzureSMR-package.R b/R/AzureSMR-package.R index d73bae2..a26592b 100644 --- a/R/AzureSMR-package.R +++ b/R/AzureSMR-package.R @@ -42,9 +42,6 @@ #' @importFrom httr add_headers headers content status_code http_status authenticate #' @importFrom httr GET PUT DELETE POST #' @importFrom XML htmlParse xpathApply xpathSApply xmlValue -#' -#' @import dplyr -#' @import magrittr #' @importFrom lubridate hour minute second #' NULL