diff --git a/DESCRIPTION b/DESCRIPTION index ef0c2d2..14c79ed 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,8 +2,8 @@ Package: AzureSMR Title: Manage and Interact with Azure Resources Description: Helps users to manage Azure Services and objects from within an R Session. This includes Azure Storage (e.g. containers and blobs), Virtual - Machines and HDInsight (Spark, Hive). To use the package, you must configure - an Azure Active Directory application and service principal in the Azure portal. + Machines and HDInsight (Spark, Hive). To use the package, you must configure an + Azure Active Directory application and service principal in the Azure portal. Type: Package Version: 0.2.5 Date: 2017-06-06 @@ -18,16 +18,17 @@ URL: https://github.com/Microsoft/AzureSMR BugReports: https://github.com/Microsoft/AzureSMR/issues NeedsCompilation: no Imports: - assertthat, + assertthat, httr, jsonlite, XML, base64enc, digest, - shiny (>= 0.13), - miniUI (>= 0.1.1), - rstudioapi (>= 0.5), - DT + shiny (>= 0.13), + miniUI (>= 0.1.1), + rstudioapi (>= 0.5), + DT, + lubridate, Depends: R(>= 3.0.0) Suggests: diff --git a/NAMESPACE b/NAMESPACE index c553922..b6c9142 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,15 +6,18 @@ S3method(summary,azureScriptActionHistory) export(AzureListRG) export(as.azureActiveContext) export(azureAuthenticate) +export(azureBatchGetKey) export(azureBlobCD) export(azureBlobFind) export(azureBlobLS) export(azureCancelDeploy) export(azureCheckToken) +export(azureCreateBatchAccount) export(azureCreateHDI) export(azureCreateResourceGroup) export(azureCreateStorageAccount) export(azureCreateStorageContainer) +export(azureDeleteBatchAccount) export(azureDeleteBlob) export(azureDeleteDeploy) export(azureDeleteHDI) @@ -30,6 +33,7 @@ export(azureHDIConf) export(azureHiveSQL) export(azureHiveStatus) export(azureListAllResources) +export(azureListBatchAccounts) export(azureListHDI) export(azureListRG) export(azureListSA) diff --git a/R/AzureBatch.R b/R/AzureBatch.R new file mode 100644 index 0000000..0d24165 --- /dev/null +++ b/R/AzureBatch.R @@ -0,0 +1,160 @@ +#' List batch accounts. +#' +#' @inheritParams setAzureContext +#' @inheritParams azureAuthenticate +#' @inheritParams azureBatchGetKey + +#' @family Batch account functions +#' @export +azureListBatchAccounts <- function(azureActiveContext, resourceGroup, subscriptionID, + verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) + + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + assert_that(is_subscription_id(subscriptionID)) + + type_batch <- "Microsoft.Batch/batchAccounts" + + z <- if(missing(resourceGroup)) { + azureListAllResources(azureActiveContext, type = type_batch) + } else { + azureListAllResources(azureActiveContext, type = type_batch, resourceGroup = resourceGroup, subscriptionID = subscriptionID) + } + rownames(z) <- NULL + z$batchAccount <- extractStorageAccount(z$id) + z +} + + +#' Create an azure batch account. +#' +#' @inheritParams setAzureContext +#' @inheritParams azureAuthenticate +#' @inheritParams azureBatchGetKey +#' @param location A string for the location to create batch account +#' @param asynchronous If TRUE, submits asynchronous request to Azure. Otherwise waits until batch account is created. +#' @family Batch account functions +#' @export +azureCreateBatchAccount <- function(azureActiveContext, batchAccount, + location = "northeurope", + resourceGroup, subscriptionID, + asynchronous = FALSE, verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) + + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup + assert_that(is_resource_group(resourceGroup)) + assert_that(is_subscription_id(subscriptionID)) + assert_that(is_storage_account(batchAccount)) + + body <- paste0('{ + "location":"', location, '", + }' + ) + + uri <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + "/resourceGroups/", resourceGroup, "/providers/Microsoft.Batch/batchAccounts/", + batchAccount, "?api-version=2017-05-01") + + r <- call_azure_sm(azureActiveContext, uri = uri, body = body, + verb = "PUT", verbose = verbose) + + if (status_code(r) == 409) { + message("409: Conflict : Account already exists with the same name") + return(TRUE) + } + + if (status_code(r) == 200) { + message("Account already exists with the same properties") + } + stopWithAzureError(r) + + rl <- content(r, "text", encoding = "UTF-8") + azureActiveContext$batchAccount <- batchAccount + azureActiveContext$resourceGroup <- resourceGroup + message("Create request Accepted. It can take a few moments to provision the batch account") + + if (!asynchronous) { + wait_for_azure( + batchAccount %in% azureListBatchAccounts(azureActiveContext, subscriptionID = subscriptionID)$name + ) + } + TRUE +} + + +#' Delete an azure batch account. +#' +#' @inheritParams setAzureContext +#' @inheritParams azureAuthenticate +#' @inheritParams azureBatchGetKey + +#' @family Batch account functions +#' @export +azureDeleteBatchAccount <- function(azureActiveContext, batchAccount, + resourceGroup, subscriptionID, verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) + + if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + + assert_that(is_storage_account(batchAccount)) + assert_that(is_resource_group(resourceGroup)) + assert_that(is_subscription_id(subscriptionID)) + + uri <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + "/resourceGroups/", resourceGroup, "/providers/Microsoft.Batch/batchAccounts/", + batchAccount, "?api-version=2017-05-01") + + r <- call_azure_sm(azureActiveContext, uri = uri, + verb = "DELETE", verbose = verbose) + + if (status_code(r) == 204) { + warning("Batch Account not found") + return(FALSE) + } + if (status_code(r) != 200) stopWithAzureError(r) + + azureActiveContext$batchAccount <- batchAccount + azureActiveContext$resourceGroup <- resourceGroup + TRUE +} + + +#' Get the Batch Keys for Specified Batch Account. +#' +#' @inheritParams setAzureContext +#' @inheritParams azureAuthenticate +#' +#' @family Batch account functions +#' @export +azureBatchGetKey <- function(azureActiveContext, batchAccount, + resourceGroup, subscriptionID, verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) + + if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + + assert_that(is_storage_account(batchAccount)) + assert_that(is_resource_group(resourceGroup)) + assert_that(is_subscription_id(subscriptionID)) + + message("Fetching Batch Key..") + + uri <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + "/resourceGroups/", resourceGroup, + "/providers/Microsoft.Batch/batchAccounts/", batchAccount, + "/listkeys?api-version=2017-05-01") + + r <- call_azure_sm(azureActiveContext, uri = uri, + verb = "POST", verbose = verbose) + stopWithAzureError(r) + + rl <- content(r, "text", encoding = "UTF-8") + df <- fromJSON(rl) + azureActiveContext$batchAccount <- batchAccount + azureActiveContext$resourceGroup <- resourceGroup + azureActiveContext$batchKey <- df$primary + + return(azureActiveContext$batchKey) +} \ No newline at end of file diff --git a/R/AzureCost.R b/R/AzureCost.R new file mode 100644 index 0000000..c5e7582 --- /dev/null +++ b/R/AzureCost.R @@ -0,0 +1,403 @@ +#' Get data consumption of an Azure subscription for a time period. Aggregation +#' method can be either daily based or hourly based. +#' +#' @note Formats of start time point and end time point follow ISO 8601 standard +#' Say if one would like to calculate data consumption between Feb 21, 2017 to +#' Feb 25, 2017, with an aggregation granularity of "daily based", the inputs +#' should be "2017-02-21 00:00:00" and "2017-02-25 00:00:00", for start time +#' point and end time point, respectively. If the aggregation granularity is +#' hourly based, the inputs can be "2017-02-21 01:00:00" and +#' "2017-02-21 02:00:00", for start and end time point, respectively. +#' NOTE by default the Azure data +#' consumption API does not allow an aggregation granularity that is finer +#' than an hour. In the case of "hourly based" granularity, if the time +#' difference between start and end time point is less than an hour, data +#' consumption will still be calculated hourly based with end time postponed. +#' For example, if the start time point and end time point are "2017-02-21 +#' 00:00:00" and "2017-02-21 00:45:00", the actual returned results are +#' data consumption in the interval of "2017-02-21 00:00:00" and +#' "2017-02-21 01:00:00". However this calculation is merely for retrieving +#' the information of an existing instance instance (e.g., meterId) with +#' which the pricing rate is multiplied by to obtain the overall expense. +#' Time zone of all time inputs are synchronized to UTC. +#' +#' @inheritParams setAzureContext +#' +#' @param instance Instance name that one would like to check expe +#' nse. It is by default empty, which returns data consumption for +#' all instances under subscription. +#' +#' @param timeStart Start time. +#' +#' @param timeEnd End time. +#' +#' @param granularity Aggregation granularity. Can be either "Daily" or +#' "Hourly". +#' @export +azureDataConsumption <- function(azureActiveContext, + instance="", + timeStart, + timeEnd, + granularity="Hourly", + verbose=FALSE) { + + # check the validity of credentials. + + assert_that(is.azureActiveContext(azureActiveContext)) + + # renew token if it expires. + + azureCheckToken(azureActiveContext) + + # preconditions here... + + if(missing(timeStart)) + stop("Please specify a starting time point in YYYY-MM-DD HH:MM:SS format.") + + if(missing(timeEnd)) + stop("Please specify an ending time point in YYYY-MM-DD HH:MM:SS format.") + + ds <- try(as.POSIXlt(timeStart, format= "%Y-%m-%d %H:%M:%S", tz="UTC")) + de <- try(as.POSIXlt(timeEnd, format= "%Y-%m-%d %H:%M:%S", tz="UTC")) + + if (class(ds) == "try-error" || + is.na(ds) || + class(de) == "try-error" || + is.na(de)) + stop("Input date format should be YYYY-MM-DD HH:MM:SS.") + + timeStart <- ds + timeEnd <- de + + if (timeStart >= timeEnd) + stop("End time is no later than start time!") + + lubridate::minute(timeStart) <- 0 + lubridate::second(timeStart) <- 0 + lubridate::minute(timeEnd) <- 0 + lubridate::second(timeEnd) <- 0 + + if (granularity == "Daily") { + + # timeStart and timeEnd should be some day at midnight. + + lubridate::hour(timeStart) <- 0 + lubridate::hour(timeEnd) <- 0 + + } + + # If the computation time is less than a hour, timeEnd will be incremented by + # an hour to get the total cost within an hour aggregated from timeStart. + # However, only the consumption on computation is considered in the returned + # data, and the computation consumption will then be replaced with the actual + # timeEnd - timeStart. + + # NOTE: estimation of cost in this case is rough though, it captures the major + # component of total cost, which originates from running an Azure instance. + # Other than computation cost, there are also cost on activities such as data + # transfer, software library license, etc. This is not included in the + # approximation here until a solid method for capturing those consumption data + # is found. Data ingress does not generate cost, but data egress does. Usually + # the occurrence of data transfer is not that frequent as computation, and + # pricing rates for data transfer is also less than computation (e.g., price + # rate of "data transfer in" is ~ 40% of that of computation on an A3 virtual + # machine). + + # TODO: inlude other types of cost for jobs that take less than an hour. + + if (as.numeric(timeEnd - timeStart) == 0) { + writeLines("Difference between timeStart and timeEnd is less than the + aggregation granularity. Cost is estimated solely on computation + running time.") + + # increment timeEnd by one hour. + + timeEnd <- timeEnd + 3600 + } + + # reformat time variables to make them compatible with API call. + + start <- URLencode(paste(as.Date(timeStart), + "T", + sprintf("%02d", lubridate::hour(timeStart)), + ":", + sprintf("%02d", lubridate::minute(timeStart)), + ":", + sprintf("%02d", lubridate::second(timeStart)), + "+", + "00:00", + sep=""), + reserved=TRUE) + + end <- URLencode(paste(as.Date(timeEnd), + "T", + sprintf("%02d", lubridate::hour(timeEnd)), + ":", + sprintf("%02d", lubridate::minute(timeEnd)), + ":", + sprintf("%02d", lubridate::second(timeEnd)), + "+", + "00:00", + sep=""), + reserved=TRUE) + + url <- + sprintf("https://management.azure.com/subscriptions/%s/providers/ + Microsoft.Commerce/UsageAggregates?api-version=%s + &reportedStartTime=%s&reportedEndTime=%s + &aggregationgranularity=%s&showDetails=%s", + azureActiveContext$subscriptionID, + "2015-06-01-preview", + start, + end, + granularity, + "false" + ) + + r <- call_azure_sm(azureActiveContext, + uri=url, + verb="GET", + verbose=verbose) + + stopWithAzureError(r) + + rl <- content(r, "text", encoding="UTF-8") + + df <- fromJSON(rl) + + df_use <- df$value$properties + + inst_data <- lapply(df$value$properties$instanceData, fromJSON) + + # retrieve results that match instance name. + + if (instance != "") { + instance_detect <- function(inst_data) { + return(basename(inst_data$Microsoft.Resources$resourceUri) == instance) + } + + index_instance <- which(unlist(lapply(inst_data, instance_detect))) + + if(!missing(instance)) { + if(length(index_instance) == 0) + stop("No data consumption records found for the instance during the + given period.") + df_use <- df_use[index_instance, ] + } else if(missing(instance)) { + if(length(index_resource) == 0) + stop("No data consumption records found for the resource group during + the given period.") + df_use <- df_use[index_resource, ] + } + } + + # if time difference is less than one hour. Only return one row of computation + # consumption whose value is the time difference. + + # timeEnd <- timeEnd - 3600 + + if(as.numeric(timeEnd - timeStart) == 0) { + + time_diff <- as.numeric(de - ds) / 3600 + + df_use <- df_use[which(df_use$meterName == "Compute Hours"), ] + df_use <- df_use[1, ] + + df_use$quantity <- df_use$time_diff + + } else { + + # NOTE the maximum number of records returned from API is limited to 1000. + + if (nrow(df_use) == 1000 && + max(as.POSIXct(df_use$usageEndTime)) < as.POSIXct(end)) { + warning(sprintf("The number of records in the specified time period %s + to %s exceeds the limit that can be returned from API call. + Consumption information is truncated. Please use a small + period instead.", timeStart, timeEnd)) + } + } + + df_use <- df_use[, c("usageStartTime", + "usageEndTime", + "meterName", + "meterCategory", + "meterSubCategory", + "unit", + "meterId", + "quantity", + "meterRegion")] + + df_use$usageStartTime <- as.POSIXct(df_use$usageStartTime) + df_use$usageEndTime <- as.POSIXct(df_use$usageEndTime) + + writeLines(sprintf("The data consumption for %s between %s and %s is", + instance, + as.character(timeStart), + as.character(timeEnd))) + + return(df_use) +} + +#' Get pricing details of resources under a subscription. +#' +#' @inheritParams setAzureContext +#' +#' @param currency Currency in which price rating is measured. +#' +#' @param locale Locality information of subscription. +#' +#' @param offerId Offer ID of the subscription. Detailed information can be +#' found at https://azure.microsoft.com/en-us/support/legal/offer-details/ +#' +#' @param region region information about the subscription. +#' +#' @note The pricing rates function wraps API calls to Azure RateCard and +#' current only the API supports only for Pay-As-You-Go offer scheme. +#' +#' @export +azurePricingRates <- function(azureActiveContext, + currency, + locale, + offerId, + region, + verbose=FALSE +) { + # renew token if it expires. + + azureCheckToken(azureActiveContext) + + # preconditions. + + if(missing(currency)) + stop("Error: please provide currency information.") + + if(missing(locale)) + stop("Error: please provide locale information.") + + if(missing(offerId)) + stop("Error: please provide offer ID.") + + if(missing(region)) + stop("Error: please provide region information.") + + url <- paste( + "https://management.azure.com/subscriptions/", + azureActiveContext$subscriptionID, + "/providers/Microsoft.Commerce/RateCard?api-version=2016-08-31-preview& + $filter=", + "OfferDurableId eq '", offerId, "'", + " and Currency eq '", currency, "'", + " and Locale eq '", locale, "'", + " and RegionInfo eq '", region, "'", + sep="") + + url <- URLencode(url) + + r <- call_azure_sm(azureActiveContext, + uri=url, + verb="GET", + verbose=verbose) + + stopWithAzureError(r) + + rl <- fromJSON(content(r, "text", encoding="UTF-8"), simplifyDataFrame=TRUE) + + df_meter <- rl$Meters + df_meter$MeterRate <- rl$Meters$MeterRates$`0` + + # NOTE: an irresponsible drop of MeterRates and MeterTags. Will add them back + # after having a better handle of them. + + df_meter <- subset(df_meter, select=-MeterRates) + df_meter <- subset(df_meter, select=-MeterTags) + + names(df_meter) <- paste0(tolower(substring(names(df_meter), + 1, + 1)), + substring(names(df_meter), 2)) + + df_meter +} + +#' Calculate cost of using a specific instance of Azure for certain period. +#' +#' @inheritParams setAzureContext +#' +#' @inheritParams azureDataConsumption +#' +#' @inheritParams azurePricingRates +#' +#' @return Total cost measured in the given currency of the specified Azure +#' instance in the period. +#' +#' @note Note if difference between \code{timeStart} and \code{timeEnd} is +#' less than the finest granularity, e.g., "Hourly" (we notice this is a +#' usual case when one needs to be aware of the charges of a job that takes +#' less than an hour), the expense will be estimated based solely on computation +#' hour. That is, the total expense is the multiplication of computation hour +#' and pricing rate of the requested instance. +#' +#' @export +azureExpenseCalculator <- function(azureActiveContext, + instance="", + timeStart, + timeEnd, + granularity, + currency, + locale, + offerId, + region, + verbose=FALSE) { + df_use <- azureDataConsumption(azureActiveContext, + instance=instance, + timeStart=timeStart, + timeEnd=timeEnd, + granularity=granularity, + verbose=verbose) + + df_used_data <- df_use[, c("meterId", + "meterSubCategory", + "usageStartTime", + "usageEndTime", + "quantity")] + + # use meterId to find pricing rates and then calculate total cost. + + df_rates <- azurePricingRates(azureActiveContext, + currency=currency, + locale=locale, + region=region, + offerId=offerId, + verbose=verbose) + + meter_list <- unique(df_used_data$meterId) + + df_used_rates <- df_rates[which(df_rates$meterId %in% meter_list), ] + df_used_rates$meterId <- df_used_rates$meterId + + # join data consumption and meter pricing rate. + + df_merged <- merge(x=df_used_data, + y=df_used_rates, + by="meterId", + all.x=TRUE) + + df_merged$meterSubCategory <- df_merged$meterSubCategory.y + df_merged$cost <- df_merged$quantity * df_merged$meterRate + + df_cost <- df_merged[, c("meterName", + "meterCategory", + "meterSubCategory", + "quantity", + "unit", + "meterRate", + "cost")] + + names(df_cost) <- paste0(tolower(substring(names(df_cost), + 1, + 1)), + substring(names(df_cost), 2)) + + df_cost +} diff --git a/R/AzureSMR-package.R b/R/AzureSMR-package.R index 81111a0..de98540 100644 --- a/R/AzureSMR-package.R +++ b/R/AzureSMR-package.R @@ -28,6 +28,11 @@ #' - [azureDeleteHDI()] #' - [azureRunScriptAction()] #' - [azureScriptActionHistory()] +#' * Azure batch: +#' - [azureListBatchAccounts()] +#' - [azureCreateBatchAccount()] +#' - [azureDeleteBatchAccount()] +#' - [azureBatchGetKey()] #' #' #' @name AzureSMR @@ -42,5 +47,6 @@ #' @importFrom httr add_headers headers content status_code http_status authenticate #' @importFrom httr GET PUT DELETE POST #' @importFrom XML htmlParse xpathApply xpathSApply xmlValue +#' @importFrom lubridate hour minute second #' NULL diff --git a/R/hdi_json.R b/R/hdi_json.R index 9c3db08..c652b5a 100644 --- a/R/hdi_json.R +++ b/R/hdi_json.R @@ -92,17 +92,17 @@ paste0(' hive_json = function(hiveServer, hiveDB, hiveUser, hivePassword) { paste0(' - ",hive-site": { - "javax.jdo.option.ConnectionDrivername": "com.microsoft.sqlserver.jdbc.SQLServerDriver", + ,"hive-site": { + "javax.jdo.option.ConnectionDriverName": "com.microsoft.sqlserver.jdbc.SQLServerDriver", "javax.jdo.option.ConnectionURL":"jdbc:sqlserver://', hiveServer, ';database=', hiveDB, ';encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", - "javax.jdo.option.ConnectionUsername":"', hiveUser, '", + "javax.jdo.option.ConnectionUserName":"', hiveUser, '", "javax.jdo.option.ConnectionPassword":"', hivePassword, '" }, "hive-env": { "hive_database": "Existing MSSQL Server database with SQL authentication", - "hive_database_name": "HIVEDB", + "hive_database_name": "', hiveDB, '", "hive_database_type": "mssql", - "hive_existing_mssql_server_database": "HIVEDB", + "hive_existing_mssql_server_database": "', hiveDB, '", "hive_existing_mssql_server_host":"', hiveServer, '", "hive_hostname":"', hiveServer,'" }' diff --git a/R/internal.R b/R/internal.R index aacc788..aed841d 100644 --- a/R/internal.R +++ b/R/internal.R @@ -87,15 +87,6 @@ createAzureStorageSignature <- function(url, verb, ) } - -#x_ms_date <- function() { - #english <- "English_United Kingdom.1252" - #old_locale <- Sys.getlocale(category = "LC_TIME") - #on.exit(Sys.setlocale(locale = old_locale)) - #Sys.setlocale(category = "LC_TIME", locale = english) - #strftime(Sys.time(), "%a, %d %b %Y %H:%M:%S %Z", tz = "GMT") -#} - x_ms_date <- function() httr::http_date(Sys.time()) azure_storage_header <- function(shared_key, date = x_ms_date(), content_length = 0) { @@ -174,7 +165,7 @@ refreshStorageKey <- function(azureActiveContext, storageAccount, resourceGroup) updateAzureActiveContext <- function(x, storageAccount, storageKey, resourceGroup, container, blob, directory) { # updates the active azure context in place - if (!is.azureActiveContext(x)) return(FALSE) + assert_that(is.azureActiveContext(azureActiveContext)) if (!missing(storageAccount)) x$storageAccount <- storageAccount if (!missing(resourceGroup)) x$resourceGroup <- resourceGroup if (!missing(storageKey)) x$storageKey <- storageKey @@ -184,24 +175,3 @@ updateAzureActiveContext <- function(x, storageAccount, storageKey, resourceGrou TRUE } -validateStorageArguments <- function(resourceGroup, storageAccount, container, storageKey) { - msg <- character(0) - pasten <- function(x, ...) paste(x, ..., collapse = "", sep = "\n") - if (!missing(resourceGroup) && (is.null(resourceGroup) || length(resourceGroup) == 0)) { - msg <- pasten(msg, "- No resourceGroup provided. Use resourceGroup argument or set in AzureContext") - } - if (!missing(storageAccount) && (is.null(storageAccount) || length(storageAccount) == 0)) { - msg <- pasten(msg, "- No storageAccount provided. Use storageAccount argument or set in AzureContext") - } - if (!missing(container) && (is.null(container) || length(container) == 0)) { - msg <- pasten(msg, "- No container provided. Use container argument or set in AzureContext") - } - if (!missing(storageKey) && (is.null(storageKey) || length(storageKey) == 0)) { - msg <- pasten(msg, "- No storageKey provided. Use storageKey argument or set in AzureContext") - } - - if (length(msg) > 0) { - stop(msg, call. = FALSE) - } - msg -} diff --git a/R/zzz.R b/R/zzz.R index 6c8ef31..3c604e3 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,4 +1,6 @@ -AzureSMR.config.default <- "~/.azuresmr/config.json" +AzureSMR.config.default <- ifelse(Sys.info()["sysname"] == "Windows", + paste0("C:/Users/", Sys.getenv("USERNAME"), "/.azuresmr/config.json"), + "~/.azuresmr/config.json") .onAttach <- function(libname, pkgname) { if (is.null(getOption("AzureSMR.config"))) diff --git a/man/AzureSMR.Rd b/man/AzureSMR.Rd index 8e4a5f9..2c8a0bc 100644 --- a/man/AzureSMR.Rd +++ b/man/AzureSMR.Rd @@ -46,6 +46,13 @@ This enables you to use and change many Azure resources. The following is an inc \item \code{\link[=azureRunScriptAction]{azureRunScriptAction()}} \item \code{\link[=azureScriptActionHistory]{azureScriptActionHistory()}} } +\item Azure batch: +\itemize{ +\item \code{\link[=azureListBatchAccounts]{azureListBatchAccounts()}} +\item \code{\link[=azureCreateBatchAccount]{azureCreateBatchAccount()}} +\item \code{\link[=azureDeleteBatchAccount]{azureDeleteBatchAccount()}} +\item \code{\link[=azureBatchGetKey]{azureBatchGetKey()}} +} } } \keyword{package} diff --git a/man/azureBatchGetKey.Rd b/man/azureBatchGetKey.Rd new file mode 100644 index 0000000..e8fc7d5 --- /dev/null +++ b/man/azureBatchGetKey.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureBatch.R +\name{azureBatchGetKey} +\alias{azureBatchGetKey} +\title{Get the Batch Keys for Specified Batch Account.} +\usage{ +azureBatchGetKey(azureActiveContext, batchAccount, resourceGroup, + subscriptionID, verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{resourceGroup}{Name of the resource group} + +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} + +\item{verbose}{Print Tracing information (Default False)} +} +\description{ +Get the Batch Keys for Specified Batch Account. +} +\seealso{ +Other Batch account functions: \code{\link{azureCreateBatchAccount}}, + \code{\link{azureDeleteBatchAccount}}, + \code{\link{azureListBatchAccounts}} +} diff --git a/man/azureCreateBatchAccount.Rd b/man/azureCreateBatchAccount.Rd new file mode 100644 index 0000000..c027f3d --- /dev/null +++ b/man/azureCreateBatchAccount.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureBatch.R +\name{azureCreateBatchAccount} +\alias{azureCreateBatchAccount} +\title{Create an azure batch account.} +\usage{ +azureCreateBatchAccount(azureActiveContext, batchAccount, + location = "northeurope", resourceGroup, subscriptionID, + asynchronous = FALSE, verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{location}{A string for the location to create batch account} + +\item{resourceGroup}{Name of the resource group} + +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} + +\item{asynchronous}{If TRUE, submits asynchronous request to Azure. Otherwise waits until batch account is created.} + +\item{verbose}{Print Tracing information (Default False)} +} +\description{ +Create an azure batch account. +} +\seealso{ +Other Batch account functions: \code{\link{azureBatchGetKey}}, + \code{\link{azureDeleteBatchAccount}}, + \code{\link{azureListBatchAccounts}} +} diff --git a/man/azureDataConsumption.Rd b/man/azureDataConsumption.Rd new file mode 100644 index 0000000..c2ee776 --- /dev/null +++ b/man/azureDataConsumption.Rd @@ -0,0 +1,49 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureCost.R +\name{azureDataConsumption} +\alias{azureDataConsumption} +\title{Get data consumption of an Azure subscription for a time period. Aggregation +method can be either daily based or hourly based.} +\usage{ +azureDataConsumption(azureActiveContext, instance = "", timeStart, timeEnd, + granularity = "Hourly", verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{instance}{Instance name that one would like to check expe +nse. It is by default empty, which returns data consumption for +all instances under subscription.} + +\item{timeStart}{Start time.} + +\item{timeEnd}{End time.} + +\item{granularity}{Aggregation granularity. Can be either "Daily" or +"Hourly".} +} +\description{ +Get data consumption of an Azure subscription for a time period. Aggregation +method can be either daily based or hourly based. +} +\note{ +Formats of start time point and end time point follow ISO 8601 standard +Say if one would like to calculate data consumption between Feb 21, 2017 to +Feb 25, 2017, with an aggregation granularity of "daily based", the inputs +should be "2017-02-21 00:00:00" and "2017-02-25 00:00:00", for start time +point and end time point, respectively. If the aggregation granularity is +hourly based, the inputs can be "2017-02-21 01:00:00" and +"2017-02-21 02:00:00", for start and end time point, respectively. +NOTE by default the Azure data +consumption API does not allow an aggregation granularity that is finer +than an hour. In the case of "hourly based" granularity, if the time +difference between start and end time point is less than an hour, data +consumption will still be calculated hourly based with end time postponed. +For example, if the start time point and end time point are "2017-02-21 +00:00:00" and "2017-02-21 00:45:00", the actual returned results are +data consumption in the interval of "2017-02-21 00:00:00" and +"2017-02-21 01:00:00". However this calculation is merely for retrieving +the information of an existing instance instance (e.g., meterId) with +which the pricing rate is multiplied by to obtain the overall expense. +Time zone of all time inputs are synchronized to UTC. +} diff --git a/man/azureDeleteBatchAccount.Rd b/man/azureDeleteBatchAccount.Rd new file mode 100644 index 0000000..5ebcb17 --- /dev/null +++ b/man/azureDeleteBatchAccount.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureBatch.R +\name{azureDeleteBatchAccount} +\alias{azureDeleteBatchAccount} +\title{Delete an azure batch account.} +\usage{ +azureDeleteBatchAccount(azureActiveContext, batchAccount, resourceGroup, + subscriptionID, verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{resourceGroup}{Name of the resource group} + +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} + +\item{verbose}{Print Tracing information (Default False)} +} +\description{ +Delete an azure batch account. +} +\seealso{ +Other Batch account functions: \code{\link{azureBatchGetKey}}, + \code{\link{azureCreateBatchAccount}}, + \code{\link{azureListBatchAccounts}} +} diff --git a/man/azureExpenseCalculator.Rd b/man/azureExpenseCalculator.Rd new file mode 100644 index 0000000..4bd479f --- /dev/null +++ b/man/azureExpenseCalculator.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureCost.R +\name{azureExpenseCalculator} +\alias{azureExpenseCalculator} +\title{Calculate cost of using a specific instance of Azure for certain period.} +\usage{ +azureExpenseCalculator(azureActiveContext, instance = "", timeStart, timeEnd, + granularity, currency, locale, offerId, region, verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{instance}{Instance name that one would like to check expe +nse. It is by default empty, which returns data consumption for +all instances under subscription.} + +\item{timeStart}{Start time.} + +\item{timeEnd}{End time.} + +\item{granularity}{Aggregation granularity. Can be either "Daily" or +"Hourly".} + +\item{currency}{Currency in which price rating is measured.} + +\item{locale}{Locality information of subscription.} + +\item{offerId}{Offer ID of the subscription. Detailed information can be +found at https://azure.microsoft.com/en-us/support/legal/offer-details/} + +\item{region}{region information about the subscription.} +} +\value{ +Total cost measured in the given currency of the specified Azure +instance in the period. +} +\description{ +Calculate cost of using a specific instance of Azure for certain period. +} +\note{ +Note if difference between \code{timeStart} and \code{timeEnd} is +less than the finest granularity, e.g., "Hourly" (we notice this is a +usual case when one needs to be aware of the charges of a job that takes +less than an hour), the expense will be estimated based solely on computation +hour. That is, the total expense is the multiplication of computation hour +and pricing rate of the requested instance. +} diff --git a/man/azureListBatchAccounts.Rd b/man/azureListBatchAccounts.Rd new file mode 100644 index 0000000..181f31f --- /dev/null +++ b/man/azureListBatchAccounts.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureBatch.R +\name{azureListBatchAccounts} +\alias{azureListBatchAccounts} +\title{List batch accounts.} +\usage{ +azureListBatchAccounts(azureActiveContext, resourceGroup, subscriptionID, + verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{resourceGroup}{Name of the resource group} + +\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} + +\item{verbose}{Print Tracing information (Default False)} +} +\description{ +List batch accounts. +} +\seealso{ +Other Batch account functions: \code{\link{azureBatchGetKey}}, + \code{\link{azureCreateBatchAccount}}, + \code{\link{azureDeleteBatchAccount}} +} diff --git a/man/azurePricingRates.Rd b/man/azurePricingRates.Rd new file mode 100644 index 0000000..c3e678d --- /dev/null +++ b/man/azurePricingRates.Rd @@ -0,0 +1,28 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AzureCost.R +\name{azurePricingRates} +\alias{azurePricingRates} +\title{Get pricing details of resources under a subscription.} +\usage{ +azurePricingRates(azureActiveContext, currency, locale, offerId, region, + verbose = FALSE) +} +\arguments{ +\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} + +\item{currency}{Currency in which price rating is measured.} + +\item{locale}{Locality information of subscription.} + +\item{offerId}{Offer ID of the subscription. Detailed information can be +found at https://azure.microsoft.com/en-us/support/legal/offer-details/} + +\item{region}{region information about the subscription.} +} +\description{ +Get pricing details of resources under a subscription. +} +\note{ +The pricing rates function wraps API calls to Azure RateCard and +current only the API supports only for Pay-As-You-Go offer scheme. +} diff --git a/tests/testthat/test-batch.R b/tests/testthat/test-batch.R new file mode 100644 index 0000000..44d4513 --- /dev/null +++ b/tests/testthat/test-batch.R @@ -0,0 +1,82 @@ +if (interactive()) library("testthat") + +settingsfile <- system.file("tests/testthat/config.json", package = "AzureSMR") +config <- read.AzureSMR.config(settingsfile) + +# ------------------------------------------------------------------------ + +context("Batch") + +asc <- createAzureContext() +with(config, + setAzureContext(asc, tenantID = tenantID, clientID = clientID, authKey = authKey) +) + +azureAuthenticate(asc) + +timestamp <- format(Sys.time(), format = "%y%m%d%H%M") +resourceGroup_name <- paste0("AzureSMtest_", timestamp) +batch_account <- paste0("azuresmr", timestamp) +batch_location = "westeurope" + +test_that("Can create resource group", { + skip_if_missing_config(settingsfile) + + res <- azureCreateResourceGroup(asc, location = "westeurope", resourceGroup = resourceGroup_name) + expect_equal(res, TRUE) + + wait_for_azure( + resourceGroup_name %in% azureListRG(asc)$resourceGroup + ) + expect_true(resourceGroup_name %in% azureListRG(asc)$resourceGroup) +}) + + +context(" - batch account") +test_that("create batch account", { + skip_if_missing_config(settingsfile) + + res <- azureCreateBatchAccount(asc, + batchAccount = batch_account, + resourceGroup = resourceGroup_name, + location = batch_location) + + if(res == "Account already exists with the same name") skip("Account already exists with the same name") + expect_equal(res, TRUE) + + wait_for_azure( + batch_account %in% azureListBatchAccounts(asc)$name + ) + + expect_true(batch_account %in% azureListBatchAccounts(asc)$name) +}) + +context(" - batch account list keys") +test_that("list keys", { + skip_if_missing_config(settingsfile) + + wait_for_azure( + batch_account %in% azureListBatchAccounts(asc)$name + ) + + res <- azureBatchGetKey(asc, + batchAccount = batch_account, + resourceGroup = resourceGroup_name) + + expect_true(is_storage_key(res)) +}) + +context(" - delete batch account") +test_that("can delete batch account", { + skip_if_missing_config(settingsfile) + + # delete the actual batch account + expect_true( + azureDeleteBatchAccount(asc, + batchAccount = batch_account, + resourceGroup = resourceGroup_name, + subscriptionID = asc$subscriptionID) + ) + + azureDeleteResourceGroup(asc, resourceGroup = resourceGroup_name) +}) \ No newline at end of file diff --git a/tests/testthat/test-cost.R b/tests/testthat/test-cost.R new file mode 100644 index 0000000..643e7b5 --- /dev/null +++ b/tests/testthat/test-cost.R @@ -0,0 +1,103 @@ +# ----------------------------------------------------------------- +# Test for cost functions. +# ----------------------------------------------------------------- + +# preambles. + +if (interactive()) library("testthat") + +settingsfile <- getOption("AzureSMR.config") +config <- read.AzureSMR.config() + +# setup. + +context("Data consumption and cost") + +asc <- createAzureContext() +with(config, + setAzureContext(asc, tenantID=tenantID, clientID=clientID, authKey=authKey) +) +azureAuthenticate(asc) + +timestamp <- format(Sys.time(), format="%y%m%d%H%M") +resourceGroup_name <- paste0("AzureSMtest_", timestamp) +sa_name <- paste0("azuresmr", timestamp) + +# run test. + +# get data consumption by day. + +test_that("Get data consumption by day", { + skip_if_missing_config(settingsfile) + + time_end <- paste0(as.Date(Sys.Date()), "00:00:00") + time_start <- paste0(as.Date(Sys.Date() - 365), "00:00:00") + + res <- azureDataConsumption(azureActiveContext=asc, + timeStart=time_start, + timeEnd=time_end, + granularity="Daily") + + expect_is(res, class="data.frame") + expect_identical(object=names(res), expected=c("usageStartTime", + "usageEndTime", + "meterName", + "meterCategory", + "meterSubCategory", + "unit", + "meterId", + "quantity", + "meterRegion")) +}) + +# get pricing rates for meters under subscription. + +test_that("Get pricing rates", { + skip_if_missing_config(settingsfile) + + res <- azurePricingRates(azureActiveContext=asc, + currency=config$CURRENCY, + locale=config$LOCALE, + offerId=config$OFFER, + region=config$REGION) + + expect_is(res, class="data.frame") + expect_identical(object=names(res), expected=c("effectiveDate", + "includedQuantity", + "meterCategory", + "meterId", + "meterName", + "meterRegion", + "meterStatus", + "meterSubCategory", + "unit", + "meterRate")) +}) + + +# total expense by day. + +test_that("Get cost by day", { + skip_if_missing_config(settingsfile) + + time_end <- paste0(as.Date(Sys.Date()), "00:00:00") + time_start <- paste0(as.Date(Sys.Date() - 365), "00:00:00") + + res <- azureExpenseCalculator(azureActiveContext=asc, + timeStart=time_start, + timeEnd=time_end, + granularity="Daily", + currency=config$CURRENCY, + locale=config$LOCALE, + offerId=config$OFFER, + region=config$REGION) + + expect_is(res, class="data.frame") + expect_identical(object=names(res), expected=c("meterName", + "meterCategory", + "meterSubCategory", + "quantity", + "unit", + "meterRate", + "cost")) +}) \ No newline at end of file