From 2c6721d5c07a329a1466de896a8a337c77ec3f90 Mon Sep 17 00:00:00 2001 From: kemihak Date: Thu, 19 Sep 2024 14:48:52 +0200 Subject: [PATCH 01/24] Old branch ant1717 from new master to avoid a lot of conflicts --- NAMESPACE | 1 + R/importOutput.R | 73 +++++++++++++++++++++++ R/readAntares.R | 132 ++++++++++++++++++++++++++++++++++++++++++ man/readAntaresAPI.Rd | 43 ++++++++++++++ 4 files changed, 249 insertions(+) create mode 100644 man/readAntaresAPI.Rd diff --git a/NAMESPACE b/NAMESPACE index 8bedfaff..f3fe700f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -42,6 +42,7 @@ export(mergeDigests) export(parAggregateMCall) export(ponderateMcAggregation) export(readAntares) +export(readAntaresAPI) export(readAntaresAreas) export(readAntaresClusters) export(readAntaresSTClusters) diff --git a/R/importOutput.R b/R/importOutput.R index a944e5fd..fa9ec536 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -265,6 +265,44 @@ ) } + +.api_get_aggregate_areas <- function(areas, timeStep, query_file, select, mcYears, opts) { + + endpoint_root <- paste0(opts$study_id, "/areas/aggregate/mc-ind/", opts$simOutputName, "?format=csv") + + # area + areas_url <- paste0("&areas_ids=", paste0(areas, collapse = ",")) + + # frequency + frequency_url <- paste0("&frequency=", timeStep) + + # columns + columns_url <- paste0("&columns_names=", paste0(select, collapse = ",")) + + # query file + query_file_url <- paste0("&query_file=", query_file) + + # MC years file + mc_years_url <- paste0("&mc_years=", paste0(mcYears, collapse = ",")) + + endpoint <- paste0(endpoint_root, query_file_url, frequency_url, columns_url, areas_url, mc_years_url) + res <- api_get(opts = opts, + endpoint = endpoint, + default_endpoint = "v1/studies") + + # First column is the id of the row + res <- as.data.table(res) + nb_cols <- ncol(res) + if (nb_cols > 1) { + idx_cols_to_keep <- seq(2,nb_cols) + } else { + idx_cols_to_keep <- 1 + } + + return(res[,.SD,.SDcols = idx_cols_to_keep]) +} + + .importOutputForDistricts <- function(districts, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { if (is.null(districts)) return(NULL) @@ -734,6 +772,41 @@ ) } + +.api_get_aggregate_links <- function(links, timeStep, select, mcYears, opts) { + + endpoint_root <- paste0(opts$study_id, "/links/aggregate/mc-ind/", opts$simOutputName, "?format=csv&query_file=values") + + # link + links_url <- paste0("&links_ids=", paste0(links, collapse = ",")) + + # frequency + frequency_url <- paste0("&frequency=", timeStep) + + # columns + columns_url <- paste0("&columns_names=", paste0(select, collapse = ",")) + + # MC years file + mc_years_url <- paste0("&mc_years=", paste0(mcYears, collapse = ",")) + + endpoint <- paste0(endpoint_root, frequency_url, columns_url, links_url, mc_years_url) + res <- api_get(opts = opts, + endpoint = endpoint, + default_endpoint = "v1/studies") + + # First column is the id of the row + res <- as.data.table(res) + nb_cols <- ncol(res) + if (nb_cols > 1) { + idx_cols_to_keep <- seq(2,nb_cols) + } else { + idx_cols_to_keep <- 1 + } + + return(res[,.SD,.SDcols = idx_cols_to_keep]) +} + + # The two following functions read input time series that are eventually # stored in output and rebuild the actual time series used in each Monte-Carlo # simulation diff --git a/R/readAntares.R b/R/readAntares.R index 58ab6448..8e3e0dbd 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -722,6 +722,138 @@ readAntares <- function(areas = NULL, links = NULL, clusters = NULL, } +#' Retrieve aggregated areas or links raw data from study output +#' +#' @param areas +#' Vector containing the names of the areas to import. If +#' \code{NULL} all areas are imported. +#' @param links +#' Vector containing the name of links to import. If \code{NULL} all +#' links are imported. +#' @param select +#' Names of the columns to be selected. +#' @param mcYears +#' Index of the Monte-Carlo years to import. If \code{NULL} all +#' Monte-Carlo years are imported. +#' @param query_file +#' Type of the data to import between +#' values, details, details-res, details-STstorage +#' @param timeStep +#' Resolution of the data to import between hourly, daily, +#' weekly, monthly or annual. +#' @template opts +#' +#' @export +readAntaresAPI <- function(areas = NULL, + links = NULL, + select = NULL, + mcYears = NULL, + query_file, + timeStep, + opts = simOptions() + ) { + + assert_that(timeStep %in% c("hourly", "daily", "weekly", "monthly", "annual"), msg = "Bad timeStep provided.") + assert_that(query_file %in% c("values", "details", "details-res", "details-STstorage"), msg = "Bad query_file provided.") + + if (!is_api_study(opts = opts)) { + stop("The study is not an API study", call. = FALSE) + } + if (!isTRUE(opts[["mode"]] == "Economy")) { + stop("The study is not in Economy mode", call. = FALSE) + } + + null_areas <- is.null(areas) + null_links <- is.null(links) + + if (null_areas & null_links) { + return() + } + if (!null_areas & !null_links) { + stop("You must choose between areas and links.") + } + if (query_file != "values" & !null_links) { + warning("Not used argument query_file for links. You will have only values in the table.") + } + # area + if (!null_areas) { + type <- "areas" + areas <- intersect(areas, opts[["areaList"]]) + + specific_areas <- switch(query_file, + "details" = opts[["areasWithClusters"]], + "details-res" = opts[["areasWithResClusters"]], + "details-STstorage" = opts[["areasWithSTClusters"]], + "values" = opts[["areaList"]], + character(0) + ) + if (!identical(specific_areas, character(0))) { + areas <- intersect(areas, specific_areas) + } + if (length(areas) == 0) { + warning("You have no area to execute the query.") + return() + } + } + + # links + if (!null_links) { + type <- "links" + links <- intersect(links, opts[["linkList"]]) + if (length(links) == 0) { + warning("You have no link to execute the query.") + return() + } + } + + # mcYear + if (!is.null(mcYears)) { + mcYears <- intersect(mcYears, opts[["mcYears"]]) + } + if (is.null(mcYears)) { + mcYears <- opts[["mcYears"]] + } + if (length(mcYears) == 0) { + warning("You have no Monte-Carlo year to execute the query.") + return() + } + + # select + if (!is.null(select)) { + select <- intersect(select, opts[["variables"]][[type]]) + } + if (is.null(select)) { + select <- opts[["variables"]][[type]] + } + if (length(select) == 0) { + warning("You have no columns in your output.") + return() + } + + if (!null_areas) { + res <- .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = query_file, + select = select, + mcYears = mcYears, + opts = opts + ) + } + + if (!null_links) { + res <- .api_get_aggregate_links(links = links, + timeStep = timeStep, + select = select, + mcYears = mcYears, + opts = opts + ) + } + + res <- .addClassAndAttributes(x = res, synthesis = FALSE, timeStep = timeStep, opts = opts, simplify = TRUE, type = type) + + return(res) +} + #' Function for preprocessing arguments areas, links, clusters and districts #' of readAntares. diff --git a/man/readAntaresAPI.Rd b/man/readAntaresAPI.Rd new file mode 100644 index 00000000..a02a01f2 --- /dev/null +++ b/man/readAntaresAPI.Rd @@ -0,0 +1,43 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readAntares.R +\name{readAntaresAPI} +\alias{readAntaresAPI} +\title{Retrieve aggregated areas or links raw data from study output} +\usage{ +readAntaresAPI( + areas = NULL, + links = NULL, + select = NULL, + mcYears = NULL, + query_file, + timeStep, + opts = simOptions() +) +} +\arguments{ +\item{areas}{Vector containing the names of the areas to import. If +\code{NULL} all areas are imported.} + +\item{links}{Vector containing the name of links to import. If \code{NULL} all +links are imported.} + +\item{select}{Names of the columns to be selected.} + +\item{mcYears}{Index of the Monte-Carlo years to import. If \code{NULL} all +Monte-Carlo years are imported.} + +\item{query_file}{Type of the data to import between +values, details, details-res, details-STstorage} + +\item{timeStep}{Resolution of the data to import between hourly, daily, +weekly, monthly or annual.} + +\item{opts}{List of simulation parameters returned by the function +\code{\link[=setSimulationPath]{setSimulationPath()}}} +} +\value{ +An updated list containing various information about the simulation. +} +\description{ +Retrieve aggregated areas or links raw data from study output +} From 489136ca40779bde2596777094a4969c3c3838fc Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 24 Sep 2024 17:46:47 +0200 Subject: [PATCH 02/24] Remove cols_to_keep cause endpoint does not send the index anymore and replace all values by none to reproduce the endpoint behaviour --- R/importOutput.R | 29 ++++++++++++++--------------- R/readAntares.R | 14 +++----------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index fa9ec536..391cb2ab 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -269,37 +269,36 @@ .api_get_aggregate_areas <- function(areas, timeStep, query_file, select, mcYears, opts) { endpoint_root <- paste0(opts$study_id, "/areas/aggregate/mc-ind/", opts$simOutputName, "?format=csv") + areas_url <- "" + columns_url <- "" + mc_years_url <- "" # area - areas_url <- paste0("&areas_ids=", paste0(areas, collapse = ",")) - + if (!identical(areas, "")) { + areas_url <- paste0("&areas_ids=", paste0(areas, collapse = ",")) + } # frequency frequency_url <- paste0("&frequency=", timeStep) # columns - columns_url <- paste0("&columns_names=", paste0(select, collapse = ",")) + if (!identical(select, "")) { + columns_url <- paste0("&columns_names=", paste0(select, collapse = ",")) + } # query file query_file_url <- paste0("&query_file=", query_file) # MC years file - mc_years_url <- paste0("&mc_years=", paste0(mcYears, collapse = ",")) + if (!identical(mcYears, "")) { + mc_years_url <- paste0("&mc_years=", paste0(mcYears, collapse = ",")) + } endpoint <- paste0(endpoint_root, query_file_url, frequency_url, columns_url, areas_url, mc_years_url) res <- api_get(opts = opts, endpoint = endpoint, default_endpoint = "v1/studies") - - # First column is the id of the row - res <- as.data.table(res) - nb_cols <- ncol(res) - if (nb_cols > 1) { - idx_cols_to_keep <- seq(2,nb_cols) - } else { - idx_cols_to_keep <- 1 - } - - return(res[,.SD,.SDcols = idx_cols_to_keep]) + + return(as.data.table(res)) } diff --git a/R/readAntares.R b/R/readAntares.R index 8e3e0dbd..42c4a7cf 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -784,7 +784,7 @@ readAntaresAPI <- function(areas = NULL, "details" = opts[["areasWithClusters"]], "details-res" = opts[["areasWithResClusters"]], "details-STstorage" = opts[["areasWithSTClusters"]], - "values" = opts[["areaList"]], + "values" = opts[["areaList"]], character(0) ) if (!identical(specific_areas, character(0))) { @@ -811,11 +811,7 @@ readAntaresAPI <- function(areas = NULL, mcYears <- intersect(mcYears, opts[["mcYears"]]) } if (is.null(mcYears)) { - mcYears <- opts[["mcYears"]] - } - if (length(mcYears) == 0) { - warning("You have no Monte-Carlo year to execute the query.") - return() + mcYears <- "" } # select @@ -823,11 +819,7 @@ readAntaresAPI <- function(areas = NULL, select <- intersect(select, opts[["variables"]][[type]]) } if (is.null(select)) { - select <- opts[["variables"]][[type]] - } - if (length(select) == 0) { - warning("You have no columns in your output.") - return() + select <- "" } if (!null_areas) { From a197ebcd0e7b7ee6e9d1f370f0302f014f504ee5 Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 24 Sep 2024 22:18:59 +0200 Subject: [PATCH 03/24] Synchronize .api_get_aggregate_links() with .api_get_aggregate_areas() --- R/importOutput.R | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 391cb2ab..e1a2e8b3 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -775,34 +775,34 @@ .api_get_aggregate_links <- function(links, timeStep, select, mcYears, opts) { endpoint_root <- paste0(opts$study_id, "/links/aggregate/mc-ind/", opts$simOutputName, "?format=csv&query_file=values") + links_url <- "" + columns_url <- "" + mc_years_url <- "" # link - links_url <- paste0("&links_ids=", paste0(links, collapse = ",")) - + if (!identical(links, "")) { + links_url <- paste0("&links_ids=", paste0(links, collapse = ",")) + } + # frequency frequency_url <- paste0("&frequency=", timeStep) # columns - columns_url <- paste0("&columns_names=", paste0(select, collapse = ",")) + if (!identical(select, "")) { + columns_url <- paste0("&columns_names=", paste0(select, collapse = ",")) + } # MC years file - mc_years_url <- paste0("&mc_years=", paste0(mcYears, collapse = ",")) + if (!identical(mcYears, "")) { + mc_years_url <- paste0("&mc_years=", paste0(mcYears, collapse = ",")) + } endpoint <- paste0(endpoint_root, frequency_url, columns_url, links_url, mc_years_url) res <- api_get(opts = opts, endpoint = endpoint, default_endpoint = "v1/studies") - # First column is the id of the row - res <- as.data.table(res) - nb_cols <- ncol(res) - if (nb_cols > 1) { - idx_cols_to_keep <- seq(2,nb_cols) - } else { - idx_cols_to_keep <- 1 - } - - return(res[,.SD,.SDcols = idx_cols_to_keep]) + return(as.data.table(res)) } From ad5093766b0c9aacff1d9385a86582b2901597aa Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 24 Sep 2024 22:20:13 +0200 Subject: [PATCH 04/24] Make arguments mcYears, select, areas or links empty if all values are selected to have a shorter url (endpoint behaviour) --- R/readAntares.R | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/R/readAntares.R b/R/readAntares.R index 42c4a7cf..755f4454 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -767,7 +767,7 @@ readAntaresAPI <- function(areas = NULL, null_links <- is.null(links) if (null_areas & null_links) { - return() + stop("You must provide areas xor links.") } if (!null_areas & !null_links) { stop("You must choose between areas and links.") @@ -779,20 +779,19 @@ readAntaresAPI <- function(areas = NULL, if (!null_areas) { type <- "areas" areas <- intersect(areas, opts[["areaList"]]) - - specific_areas <- switch(query_file, - "details" = opts[["areasWithClusters"]], - "details-res" = opts[["areasWithResClusters"]], - "details-STstorage" = opts[["areasWithSTClusters"]], - "values" = opts[["areaList"]], - character(0) - ) - if (!identical(specific_areas, character(0))) { - areas <- intersect(areas, specific_areas) - } - if (length(areas) == 0) { - warning("You have no area to execute the query.") - return() + if (identical(sort(areas), sort(opts[["areaList"]]))) { + areas <- "" + } else { + specific_areas <- switch(query_file, + "details" = opts[["areasWithClusters"]], + "details-res" = opts[["areasWithResClusters"]], + "details-STstorage" = opts[["areasWithSTClusters"]], + "values" = opts[["areaList"]], + character(0) + ) + if (!identical(specific_areas, character(0))) { + areas <- intersect(areas, specific_areas) + } } } @@ -800,15 +799,17 @@ readAntaresAPI <- function(areas = NULL, if (!null_links) { type <- "links" links <- intersect(links, opts[["linkList"]]) - if (length(links) == 0) { - warning("You have no link to execute the query.") - return() + if (identical(sort(links), sort(opts[["linkList"]]))) { + links <- "" } } # mcYear if (!is.null(mcYears)) { mcYears <- intersect(mcYears, opts[["mcYears"]]) + if (identical(sort(mcYears), sort(opts[["mcYears"]]))) { + mcYears <- "" + } } if (is.null(mcYears)) { mcYears <- "" @@ -817,6 +818,9 @@ readAntaresAPI <- function(areas = NULL, # select if (!is.null(select)) { select <- intersect(select, opts[["variables"]][[type]]) + if (identical(sort(select), sort(opts[["variables"]][[type]]))) { + select <- "" + } } if (is.null(select)) { select <- "" From 30f0a77501d335e600c6c46c6b51a16cabacb39c Mon Sep 17 00:00:00 2001 From: kemihak Date: Wed, 25 Sep 2024 11:09:58 +0200 Subject: [PATCH 05/24] select and mcYears empty if no intersection between the two sets --- R/readAntares.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/readAntares.R b/R/readAntares.R index 755f4454..77aff034 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -807,7 +807,7 @@ readAntaresAPI <- function(areas = NULL, # mcYear if (!is.null(mcYears)) { mcYears <- intersect(mcYears, opts[["mcYears"]]) - if (identical(sort(mcYears), sort(opts[["mcYears"]]))) { + if (identical(sort(mcYears), sort(opts[["mcYears"]])) | length(mcYears) == 0) { mcYears <- "" } } @@ -818,7 +818,7 @@ readAntaresAPI <- function(areas = NULL, # select if (!is.null(select)) { select <- intersect(select, opts[["variables"]][[type]]) - if (identical(sort(select), sort(opts[["variables"]][[type]]))) { + if (identical(sort(select), sort(opts[["variables"]][[type]])) | length(select) == 0) { select <- "" } } From 4962d8d1bea38796c3f75b4afe9fa185334f01cd Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 15 Oct 2024 17:42:49 +0200 Subject: [PATCH 06/24] Include mc-all query with argument synthesis --- R/importOutput.R | 22 ++++++++++++++++------ R/readAntares.R | 6 ++++++ man/readAntaresAPI.Rd | 3 +++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index e1a2e8b3..0aedbc1e 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -266,9 +266,14 @@ } -.api_get_aggregate_areas <- function(areas, timeStep, query_file, select, mcYears, opts) { - - endpoint_root <- paste0(opts$study_id, "/areas/aggregate/mc-ind/", opts$simOutputName, "?format=csv") +.api_get_aggregate_areas <- function(areas, timeStep, query_file, select, mcYears, synthesis, opts) { + + if (synthesis) { + pattern_endpoint <- "mc-all" + } else { + pattern_endpoint <- "mc-ind" + } + endpoint_root <- paste0(opts$study_id, "/areas/aggregate/", pattern_endpoint, "/", opts$simOutputName, "?format=csv") areas_url <- "" columns_url <- "" mc_years_url <- "" @@ -772,9 +777,14 @@ } -.api_get_aggregate_links <- function(links, timeStep, select, mcYears, opts) { - - endpoint_root <- paste0(opts$study_id, "/links/aggregate/mc-ind/", opts$simOutputName, "?format=csv&query_file=values") +.api_get_aggregate_links <- function(links, timeStep, select, mcYears, synthesis, opts) { + + if (synthesis) { + pattern_endpoint <- "mc-all" + } else { + pattern_endpoint <- "mc-ind" + } + endpoint_root <- paste0(opts$study_id, "/links/aggregate/", pattern_endpoint, "/", opts$simOutputName, "?format=csv&query_file=values") links_url <- "" columns_url <- "" mc_years_url <- "" diff --git a/R/readAntares.R b/R/readAntares.R index 77aff034..b02fdfcf 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -741,6 +741,8 @@ readAntares <- function(areas = NULL, links = NULL, clusters = NULL, #' @param timeStep #' Resolution of the data to import between hourly, daily, #' weekly, monthly or annual. +#' @param synthesis +#' Shoudl the query use mc-all or mc-ind. #' @template opts #' #' @export @@ -750,11 +752,13 @@ readAntaresAPI <- function(areas = NULL, mcYears = NULL, query_file, timeStep, + synthesis = TRUE, opts = simOptions() ) { assert_that(timeStep %in% c("hourly", "daily", "weekly", "monthly", "annual"), msg = "Bad timeStep provided.") assert_that(query_file %in% c("values", "details", "details-res", "details-STstorage"), msg = "Bad query_file provided.") + assert_that(inherits(x = synthesis, what = "logical"), msg = "Argument synthesis should be logical.") if (!is_api_study(opts = opts)) { stop("The study is not an API study", call. = FALSE) @@ -832,6 +836,7 @@ readAntaresAPI <- function(areas = NULL, query_file = query_file, select = select, mcYears = mcYears, + synthesis = synthesis, opts = opts ) } @@ -841,6 +846,7 @@ readAntaresAPI <- function(areas = NULL, timeStep = timeStep, select = select, mcYears = mcYears, + synthesis = synthesis, opts = opts ) } diff --git a/man/readAntaresAPI.Rd b/man/readAntaresAPI.Rd index a02a01f2..baa3550a 100644 --- a/man/readAntaresAPI.Rd +++ b/man/readAntaresAPI.Rd @@ -11,6 +11,7 @@ readAntaresAPI( mcYears = NULL, query_file, timeStep, + synthesis = TRUE, opts = simOptions() ) } @@ -32,6 +33,8 @@ values, details, details-res, details-STstorage} \item{timeStep}{Resolution of the data to import between hourly, daily, weekly, monthly or annual.} +\item{synthesis}{Shoudl the query use mc-all or mc-ind.} + \item{opts}{List of simulation parameters returned by the function \code{\link[=setSimulationPath]{setSimulationPath()}}} } From 8e9d85cdbfccc4931fa4039f660378d8701145d3 Mon Sep 17 00:00:00 2001 From: kemihak Date: Wed, 16 Oct 2024 10:58:02 +0200 Subject: [PATCH 07/24] Use synthesis variable to add attribute to the data.table --- R/readAntares.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/readAntares.R b/R/readAntares.R index b02fdfcf..3ff6705a 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -851,7 +851,7 @@ readAntaresAPI <- function(areas = NULL, ) } - res <- .addClassAndAttributes(x = res, synthesis = FALSE, timeStep = timeStep, opts = opts, simplify = TRUE, type = type) + res <- .addClassAndAttributes(x = res, synthesis = synthesis, timeStep = timeStep, opts = opts, simplify = TRUE, type = type) return(res) } From 41d956d33852282de7a22803da04083842b4edff Mon Sep 17 00:00:00 2001 From: kemihak Date: Wed, 16 Oct 2024 11:07:20 +0200 Subject: [PATCH 08/24] if synthesis is TRUE, mcYears value is useless --- R/readAntares.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/readAntares.R b/R/readAntares.R index 3ff6705a..03b57370 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -811,7 +811,7 @@ readAntaresAPI <- function(areas = NULL, # mcYear if (!is.null(mcYears)) { mcYears <- intersect(mcYears, opts[["mcYears"]]) - if (identical(sort(mcYears), sort(opts[["mcYears"]])) | length(mcYears) == 0) { + if (identical(sort(mcYears), sort(opts[["mcYears"]])) | length(mcYears) == 0 | synthesis) { mcYears <- "" } } From c29464d004bbbd021e31b5838f74c5e2a7eec3a6 Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 5 Nov 2024 14:41:18 +0100 Subject: [PATCH 09/24] remove readAntaresAPI() --- NAMESPACE | 1 - R/readAntares.R | 135 ------------------------------------------ man/readAntaresAPI.Rd | 46 -------------- 3 files changed, 182 deletions(-) delete mode 100644 man/readAntaresAPI.Rd diff --git a/NAMESPACE b/NAMESPACE index f3fe700f..8bedfaff 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -42,7 +42,6 @@ export(mergeDigests) export(parAggregateMCall) export(ponderateMcAggregation) export(readAntares) -export(readAntaresAPI) export(readAntaresAreas) export(readAntaresClusters) export(readAntaresSTClusters) diff --git a/R/readAntares.R b/R/readAntares.R index 03b57370..11f75659 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -722,141 +722,6 @@ readAntares <- function(areas = NULL, links = NULL, clusters = NULL, } -#' Retrieve aggregated areas or links raw data from study output -#' -#' @param areas -#' Vector containing the names of the areas to import. If -#' \code{NULL} all areas are imported. -#' @param links -#' Vector containing the name of links to import. If \code{NULL} all -#' links are imported. -#' @param select -#' Names of the columns to be selected. -#' @param mcYears -#' Index of the Monte-Carlo years to import. If \code{NULL} all -#' Monte-Carlo years are imported. -#' @param query_file -#' Type of the data to import between -#' values, details, details-res, details-STstorage -#' @param timeStep -#' Resolution of the data to import between hourly, daily, -#' weekly, monthly or annual. -#' @param synthesis -#' Shoudl the query use mc-all or mc-ind. -#' @template opts -#' -#' @export -readAntaresAPI <- function(areas = NULL, - links = NULL, - select = NULL, - mcYears = NULL, - query_file, - timeStep, - synthesis = TRUE, - opts = simOptions() - ) { - - assert_that(timeStep %in% c("hourly", "daily", "weekly", "monthly", "annual"), msg = "Bad timeStep provided.") - assert_that(query_file %in% c("values", "details", "details-res", "details-STstorage"), msg = "Bad query_file provided.") - assert_that(inherits(x = synthesis, what = "logical"), msg = "Argument synthesis should be logical.") - - if (!is_api_study(opts = opts)) { - stop("The study is not an API study", call. = FALSE) - } - if (!isTRUE(opts[["mode"]] == "Economy")) { - stop("The study is not in Economy mode", call. = FALSE) - } - - null_areas <- is.null(areas) - null_links <- is.null(links) - - if (null_areas & null_links) { - stop("You must provide areas xor links.") - } - if (!null_areas & !null_links) { - stop("You must choose between areas and links.") - } - if (query_file != "values" & !null_links) { - warning("Not used argument query_file for links. You will have only values in the table.") - } - # area - if (!null_areas) { - type <- "areas" - areas <- intersect(areas, opts[["areaList"]]) - if (identical(sort(areas), sort(opts[["areaList"]]))) { - areas <- "" - } else { - specific_areas <- switch(query_file, - "details" = opts[["areasWithClusters"]], - "details-res" = opts[["areasWithResClusters"]], - "details-STstorage" = opts[["areasWithSTClusters"]], - "values" = opts[["areaList"]], - character(0) - ) - if (!identical(specific_areas, character(0))) { - areas <- intersect(areas, specific_areas) - } - } - } - - # links - if (!null_links) { - type <- "links" - links <- intersect(links, opts[["linkList"]]) - if (identical(sort(links), sort(opts[["linkList"]]))) { - links <- "" - } - } - - # mcYear - if (!is.null(mcYears)) { - mcYears <- intersect(mcYears, opts[["mcYears"]]) - if (identical(sort(mcYears), sort(opts[["mcYears"]])) | length(mcYears) == 0 | synthesis) { - mcYears <- "" - } - } - if (is.null(mcYears)) { - mcYears <- "" - } - - # select - if (!is.null(select)) { - select <- intersect(select, opts[["variables"]][[type]]) - if (identical(sort(select), sort(opts[["variables"]][[type]])) | length(select) == 0) { - select <- "" - } - } - if (is.null(select)) { - select <- "" - } - - if (!null_areas) { - res <- .api_get_aggregate_areas(areas = areas, - timeStep = timeStep, - query_file = query_file, - select = select, - mcYears = mcYears, - synthesis = synthesis, - opts = opts - ) - } - - if (!null_links) { - res <- .api_get_aggregate_links(links = links, - timeStep = timeStep, - select = select, - mcYears = mcYears, - synthesis = synthesis, - opts = opts - ) - } - - res <- .addClassAndAttributes(x = res, synthesis = synthesis, timeStep = timeStep, opts = opts, simplify = TRUE, type = type) - - return(res) -} - - #' Function for preprocessing arguments areas, links, clusters and districts #' of readAntares. #' diff --git a/man/readAntaresAPI.Rd b/man/readAntaresAPI.Rd deleted file mode 100644 index baa3550a..00000000 --- a/man/readAntaresAPI.Rd +++ /dev/null @@ -1,46 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readAntares.R -\name{readAntaresAPI} -\alias{readAntaresAPI} -\title{Retrieve aggregated areas or links raw data from study output} -\usage{ -readAntaresAPI( - areas = NULL, - links = NULL, - select = NULL, - mcYears = NULL, - query_file, - timeStep, - synthesis = TRUE, - opts = simOptions() -) -} -\arguments{ -\item{areas}{Vector containing the names of the areas to import. If -\code{NULL} all areas are imported.} - -\item{links}{Vector containing the name of links to import. If \code{NULL} all -links are imported.} - -\item{select}{Names of the columns to be selected.} - -\item{mcYears}{Index of the Monte-Carlo years to import. If \code{NULL} all -Monte-Carlo years are imported.} - -\item{query_file}{Type of the data to import between -values, details, details-res, details-STstorage} - -\item{timeStep}{Resolution of the data to import between hourly, daily, -weekly, monthly or annual.} - -\item{synthesis}{Shoudl the query use mc-all or mc-ind.} - -\item{opts}{List of simulation parameters returned by the function -\code{\link[=setSimulationPath]{setSimulationPath()}}} -} -\value{ -An updated list containing various information about the simulation. -} -\description{ -Retrieve aggregated areas or links raw data from study output -} From e1739580039441961975c6be8a386b7f64abb96f Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 5 Nov 2024 14:42:02 +0100 Subject: [PATCH 10/24] Include endpoint mc-ind in .importOutputForLinks() --- R/importOutput.R | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 0aedbc1e..91fba5bc 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -770,13 +770,23 @@ #' .importOutputForLinks <- function(links, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { - suppressWarnings( - .importOutput("links", "values", "link", links, timeStep, select, - mcYears, showProgress, opts, parallel = parallel) - ) + + if (is_api_study(opts)) { + .api_get_aggregate_links(links = links, + timeStep = timeStep, + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + } else { + suppressWarnings( + .importOutput("links", "values", "link", links, timeStep, select, + mcYears, showProgress, opts, parallel = parallel) + ) + } } - .api_get_aggregate_links <- function(links, timeStep, select, mcYears, synthesis, opts) { if (synthesis) { From e432375201582320a67ffad3b3852c761fb3ec47 Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 5 Nov 2024 15:35:19 +0100 Subject: [PATCH 11/24] Include mc-ind endpoint in .importOutputForAreas(), .importOutputForResClusters() and .importOutputForSTClusters() --- R/importOutput.R | 75 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 91fba5bc..55a51f54 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -259,10 +259,22 @@ #' .importOutputForAreas <- function(areas, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { - suppressWarnings( - .importOutput("areas", "values", "area", areas, timeStep, select, - mcYears, showProgress, opts, parallel = parallel) - ) + + if (is_api_study(opts)) { + .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "values", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + } else { + suppressWarnings( + .importOutput("areas", "values", "area", areas, timeStep, select, + mcYears, showProgress, opts, parallel = parallel) + ) + } } @@ -565,16 +577,26 @@ .importOutputForResClusters <- function(areas, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { - - reshapeFun <- function(x) { - .reshape_details_file(x,file_type="details-res",opts=opts) + if (is_api_study(opts)) { + .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "details-res", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + } else { + reshapeFun <- function(x) { + .reshape_details_file(x,file_type="details-res",opts=opts) + } + + suppressWarnings( + .importOutput("areas", "details-res", "area", areas, timeStep, NULL, + mcYears, showProgress, opts, reshapeFun, sameNames = FALSE, + objectDisplayName = "clustersRe", parallel = parallel) + ) } - - suppressWarnings( - .importOutput("areas", "details-res", "area", areas, timeStep, NULL, - mcYears, showProgress, opts, reshapeFun, sameNames = FALSE, - objectDisplayName = "clustersRe", parallel = parallel) - ) } @@ -590,15 +612,26 @@ .importOutputForSTClusters <- function(areas, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { - reshapeFun <- function(x) { - .reshape_details_file(x,file_type="details-STstorage",opts=opts) + if (is_api_study(opts)) { + .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "details-STstorage", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + } else { + reshapeFun <- function(x) { + .reshape_details_file(x,file_type="details-STstorage",opts=opts) + } + + suppressWarnings( + .importOutput("areas", "details-STstorage", "area", areas, timeStep, NULL, + mcYears, showProgress, opts, reshapeFun, sameNames = FALSE, + objectDisplayName = "clustersST", parallel = parallel) + ) } - - suppressWarnings( - .importOutput("areas", "details-STstorage", "area", areas, timeStep, NULL, - mcYears, showProgress, opts, reshapeFun, sameNames = FALSE, - objectDisplayName = "clustersST", parallel = parallel) - ) } #' .importOutputForBindingConstraints From 6bae80af0fa23f2c303f4ef6351bf9c684ff510d Mon Sep 17 00:00:00 2001 From: kemihak Date: Sat, 9 Nov 2024 23:32:36 +0100 Subject: [PATCH 12/24] Format result for api endpoint return to match the .importOutput one --- R/importOutput.R | 71 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 55a51f54..3b91ca3a 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -261,14 +261,15 @@ showProgress, opts, parallel) { if (is_api_study(opts)) { - .api_get_aggregate_areas(areas = areas, - timeStep = timeStep, - query_file = "values", - select = select, - mcYears = mcYears, - synthesis = FALSE, - opts = opts - ) + res <- .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "values", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + .format_api_aggregate_result(res) } else { suppressWarnings( .importOutput("areas", "values", "area", areas, timeStep, select, @@ -319,6 +320,26 @@ } +.format_api_aggregate_result <- function(res) { + + res_cols <- colnames(res) + #To remove when endpoint will not send time anymore + if ("time" %in% res_cols) { + res[,time:=NULL] + } + + cols_to_factor <- c("area", "cluster") + cols_to_factor <- intersect(cols_to_factor, res_cols) + res[,(cols_to_factor):=lapply(.SD, as.factor), .SDcols=cols_to_factor] + + cols_to_integer <- c("timeId") + cols_to_integer <- intersect(cols_to_integer, res_cols) + res[,(cols_to_integer):=lapply(.SD, as.integer), .SDcols=cols_to_integer] + + return(res) +} + + .importOutputForDistricts <- function(districts, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { if (is.null(districts)) return(NULL) @@ -578,14 +599,15 @@ showProgress, opts, parallel) { if (is_api_study(opts)) { - .api_get_aggregate_areas(areas = areas, - timeStep = timeStep, - query_file = "details-res", - select = select, - mcYears = mcYears, - synthesis = FALSE, - opts = opts - ) + res <- .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "details-res", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + .format_api_aggregate_result(res) } else { reshapeFun <- function(x) { .reshape_details_file(x,file_type="details-res",opts=opts) @@ -613,14 +635,15 @@ showProgress, opts, parallel) { if (is_api_study(opts)) { - .api_get_aggregate_areas(areas = areas, - timeStep = timeStep, - query_file = "details-STstorage", - select = select, - mcYears = mcYears, - synthesis = FALSE, - opts = opts - ) + res <- .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "details-STstorage", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + .format_api_aggregate_result(res) } else { reshapeFun <- function(x) { .reshape_details_file(x,file_type="details-STstorage",opts=opts) From 785f01a41aae87b15db3787d2149ff56c8f05ae2 Mon Sep 17 00:00:00 2001 From: kemihak Date: Sat, 9 Nov 2024 23:43:56 +0100 Subject: [PATCH 13/24] Split .importOutputForClusters() mustrun vs no mustrun and make the right calls to endpoint --- R/importOutput.R | 238 +++++++++++++++++++++++++++++------------------ 1 file changed, 146 insertions(+), 92 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 3b91ca3a..7cf901db 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -417,60 +417,82 @@ } -#' .importOutputForClusters -#' -#' Private function used to import the output for the thermal clusters of one area -#' -#' @return -#' a data.table -#' -#' @noRd -#' -.importOutputForClusters <- function(areas, timeStep, select = NULL, mcYears = NULL, - showProgress, opts, mustRun = FALSE, parallel) { - - reshapeFun <- function(x){ - .reshape_details_file(x,file_type="details",opts=opts) - } - - if (!mustRun) { +.importOutputForClusters_wo_mustrun <- function(areas, timeStep, select = NULL, mcYears = NULL, + showProgress, opts, parallel){ + if (is_api_study(opts)) { + res <- .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "details", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + .format_api_aggregate_result(res) + } else { + reshapeFun <- function(x){ + .reshape_details_file(x,file_type="details",opts=opts) + } suppressWarnings( .importOutput("areas", "details", "area", areas, timeStep, NULL, mcYears, showProgress, opts, reshapeFun, sameNames = FALSE, objectDisplayName = "cluster", parallel = parallel) ) - - } else { - # The partial must run for a cluster is defined as: - # sum_t(min(production_t, capacity * minGenModulation_t)) - # This formula is non-linear, so if we need to get hourly data to compute - # it. - # To avoid importing large amount of data, we first check if minGenModulation - # is non null for at least one cluster. - # If we have to use hourly data, we aggregate it directly at the desired - # timestep to limit the amount of RAM required. - - # Get cluster capacity and must run mode - clusterDesc <- readClusterDesc(opts) - if(is.null(clusterDesc[["must-run"]])) - clusterDesc[["must-run"]] <- FALSE - clusterDesc[is.na(`must-run`), `must-run` := FALSE] - if (is.null(clusterDesc[["min-stable-power"]])) - clusterDesc[["min-stable-power"]] <- 0 - clusterDesc[is.na(`min-stable-power`), `min-stable-power` := 0] - clusterDesc <- clusterDesc[, .(area, cluster, - capacity = nominalcapacity * unitcount, - `min-stable-power`, - `must-run`)] - - # Are clusters in partial must run mode ? - mod <- llply(areas, .importThermalModulation, opts = opts, timeStep = "hourly") - mod <- rbindlist(mod) - - # Should we compute the partial must run ? - if (is.null(mod$minGenModulation) || all(is.na(mod$minGenModulation) | mod$minGenModulation == 0)) { - - # We should not \o/ + } +} + + +.importOutputForClusters_w_mustrun <- function(areas, timeStep, select = NULL, mcYears = NULL, + showProgress, opts, parallel) { + + is_api <- is_api_study(opts) + + reshapeFun <- function(x){ + .reshape_details_file(x,file_type="details",opts=opts) + } + + # The partial must run for a cluster is defined as: + # sum_t(min(production_t, capacity * minGenModulation_t)) + # This formula is non-linear, so if we need to get hourly data to compute + # it. + # To avoid importing large amount of data, we first check if minGenModulation + # is non null for at least one cluster. + # If we have to use hourly data, we aggregate it directly at the desired + # timestep to limit the amount of RAM required. + + # Get cluster capacity and must run mode + clusterDesc <- readClusterDesc(opts) + if (is.null(clusterDesc[["must-run"]])) { + clusterDesc[["must-run"]] <- FALSE + } + clusterDesc[is.na(`must-run`), `must-run` := FALSE] + if (is.null(clusterDesc[["min-stable-power"]])) { + clusterDesc[["min-stable-power"]] <- 0 + } + clusterDesc[is.na(`min-stable-power`), `min-stable-power` := 0] + clusterDesc <- clusterDesc[, .(area, cluster, + capacity = nominalcapacity * unitcount, + `min-stable-power`, + `must-run`)] + + # Are clusters in partial must run mode ? + mod <- llply(areas, .importThermalModulation, opts = opts, timeStep = "hourly") + mod <- rbindlist(mod) + + # Should we compute the partial must run ? + if (is.null(mod$minGenModulation) || all(is.na(mod$minGenModulation) | mod$minGenModulation == 0)) { + # We should not \o/ + if (is_api) { + res <- .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "details", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + res <- .format_api_aggregate_result(res) + } else { res <- suppressWarnings( suppressWarnings( .importOutput("areas", "details", "area", areas, timeStep, NULL, @@ -478,69 +500,101 @@ objectDisplayName = "cluster", parallel = parallel) ) ) - res[, mustRunPartial := 0L] + } + res[, mustRunPartial := 0L] + } else { + + # Worst case ! We have to ! + # + if(timeStep!="hourly"){ + warning('Hourly data will be imported to compute partial must run min(production_t, capacity * minGenModulation_t). These data will be aggregated at the desired `timeStep`. ') - } else { + #copy of warning in ChangeTimeStep + warning('Aggregation will be perform approximatively because optimization variables in ANTARES are doubles but ANTARES write only integers in TXT files, with this transformation we lose precision. If you want accurate data then you must import the corresponding data with `readAntares`') - # Worst case ! We have to ! - # - if(timeStep!="hourly"){ - warning('Hourly data will be imported to compute partial must run min(production_t, capacity * minGenModulation_t). These data will be aggregated at the desired `timeStep`. ') - - #copy of warning in ChangeTimeStep - warning('Aggregation will be perform approximatively because optimization variables in ANTARES are doubles but ANTARES write only integers in TXT files, with this transformation we lose precision. If you want accurate data then you must import the corresponding data with `readAntares`') - - messageWarningMcYears<-paste0("When mcYears is set to all or NULL : ", mcYears, " and timeStep is set to : " ,timeStep , " result for mustRun are not accurate. Hourly `synthetic` or `details` results will be aggregated at the desired `timeStep`. " ) - - if( is.null(mcYears) ){ - warning(messageWarningMcYears, call. = FALSE) - }else if (is.character(mcYears)){ - if (mcYears=="all"){ - warning(messageWarningMcYears, call. = FALSE) - } - }else if (length(mcYears) > 1){ + messageWarningMcYears<-paste0("When mcYears is set to all or NULL : ", mcYears, " and timeStep is set to : " ,timeStep , " result for mustRun are not accurate. Hourly `synthetic` or `details` results will be aggregated at the desired `timeStep`. " ) + + if( is.null(mcYears) ){ + warning(messageWarningMcYears, call. = FALSE) + }else if (is.character(mcYears)){ + if (mcYears=="all"){ warning(messageWarningMcYears, call. = FALSE) } + }else if (length(mcYears) > 1){ + warning(messageWarningMcYears, call. = FALSE) } + } + + mod[is.na(minGenModulation), minGenModulation := 0] - mod[is.na(minGenModulation), minGenModulation := 0] - - .mergeByRef(mod, clusterDesc) - mod[, mustRunPartial := minGenModulation * capacity] - - setkey(mod, area, cluster, timeId) - + .mergeByRef(mod, clusterDesc) + mod[, mustRunPartial := minGenModulation * capacity] + + setkey(mod, area, cluster, timeId) + + if (is_api) { + res <- .api_get_aggregate_areas(areas = areas, + timeStep = "hourly", + query_file = "details", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + res <- .format_api_aggregate_result(res) + } else { processFun <- function(x) { x <- reshapeFun(x) mustRunPartial <- mod[J(x$area, x$cluster, x$timeId), mustRunPartial] x[, mustRunPartial := pmin(production, mustRunPartial)] changeTimeStep(x, timeStep, "hourly", fun = "sum", opts = opts) } - res <- suppressWarnings( .importOutput("areas", "details", "area", areas, "hourly", NULL, mcYears, showProgress, opts, processFun, sameNames = FALSE, objectDisplayName = "cluster", parallel = parallel) ) - } - .mergeByRef(res, clusterDesc[,.(area, cluster, `must-run`, `min-stable-power`)]) - - if (is.null(res$NODU)) res[, thermalPmin := 0] - else res[, thermalPmin := `min-stable-power` * NODU] - - res[, `:=`( - mustRun = production * `must-run`, - mustRunTotal = production * `must-run` + mustRunPartial, - `must-run` = NULL, - `min-stable-power` = NULL - )] - - res[, thermalPmin := pmax(thermalPmin, mustRunTotal)] - - res + } + + .mergeByRef(res, clusterDesc[,.(area, cluster, `must-run`, `min-stable-power`)]) + + if (is.null(res$NODU)) res[, thermalPmin := 0] + else res[, thermalPmin := `min-stable-power` * NODU] + + res[, `:=`( + mustRun = production * `must-run`, + mustRunTotal = production * `must-run` + mustRunPartial, + `must-run` = NULL, + `min-stable-power` = NULL + )] + + res[, thermalPmin := pmax(thermalPmin, mustRunTotal)] + + res +} + + +#' .importOutputForClusters +#' +#' Private function used to import the output for the thermal clusters of one area +#' +#' @return +#' a data.table +#' +#' @noRd +#' +.importOutputForClusters <- function(areas, timeStep, select = NULL, mcYears = NULL, + showProgress, opts, mustRun = FALSE, parallel) { + + if (!mustRun) { + .importOutputForClusters_wo_mustrun(areas = areas, timeStep = timeStep, select = select, mcYears = mcYears, + showProgress = showProgress, opts = opts, parallel = parallel) + } else { + .importOutputForClusters_w_mustrun(areas = areas, timeStep = timeStep, select = select, mcYears = mcYears, + showProgress = showProgress, opts = opts, parallel = parallel) } } From c47850061afb25ed29cfe734a1e57c5d4c838fe6 Mon Sep 17 00:00:00 2001 From: kemihak Date: Sun, 10 Nov 2024 00:30:06 +0100 Subject: [PATCH 14/24] Treat NULL areas --- R/importOutput.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R/importOutput.R b/R/importOutput.R index 7cf901db..7ce2a00e 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -281,6 +281,8 @@ .api_get_aggregate_areas <- function(areas, timeStep, query_file, select, mcYears, synthesis, opts) { + if (is.null(areas)) return(NULL) + if (synthesis) { pattern_endpoint <- "mc-all" } else { @@ -322,6 +324,8 @@ .format_api_aggregate_result <- function(res) { + if (is.null(res)) return(NULL) + res_cols <- colnames(res) #To remove when endpoint will not send time anymore if ("time" %in% res_cols) { From f7c633384066d7a97a01bf8d3ed99dcabc5a7c37 Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 12 Nov 2024 12:06:12 +0100 Subject: [PATCH 15/24] Return NULl if synthesis is TRUE and integrate links part --- R/importOutput.R | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 7ce2a00e..063f9092 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -281,7 +281,7 @@ .api_get_aggregate_areas <- function(areas, timeStep, query_file, select, mcYears, synthesis, opts) { - if (is.null(areas)) return(NULL) + if (is.null(areas) | is.null(mcYears) | synthesis) return(NULL) if (synthesis) { pattern_endpoint <- "mc-all" @@ -332,7 +332,7 @@ res[,time:=NULL] } - cols_to_factor <- c("area", "cluster") + cols_to_factor <- c("area", "link", "cluster") cols_to_factor <- intersect(cols_to_factor, res_cols) res[,(cols_to_factor):=lapply(.SD, as.factor), .SDcols=cols_to_factor] @@ -885,14 +885,15 @@ .importOutputForLinks <- function(links, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { - if (is_api_study(opts)) { - .api_get_aggregate_links(links = links, - timeStep = timeStep, - select = select, - mcYears = mcYears, - synthesis = FALSE, - opts = opts - ) + if (!is_api_study(opts)) { + res <- .api_get_aggregate_links(links = links, + timeStep = timeStep, + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + .format_api_aggregate_result(res) } else { suppressWarnings( .importOutput("links", "values", "link", links, timeStep, select, @@ -903,6 +904,8 @@ .api_get_aggregate_links <- function(links, timeStep, select, mcYears, synthesis, opts) { + if (is.null(links) | is.null(mcYears) | synthesis) return(NULL) + if (synthesis) { pattern_endpoint <- "mc-all" } else { From 8378b644d865c135d8a2957f4f1d4ff2c257a219 Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 12 Nov 2024 12:07:04 +0100 Subject: [PATCH 16/24] Call endpoint if API for links --- R/importOutput.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/importOutput.R b/R/importOutput.R index 063f9092..99b6dc2c 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -885,7 +885,7 @@ .importOutputForLinks <- function(links, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { - if (!is_api_study(opts)) { + if (is_api_study(opts)) { res <- .api_get_aggregate_links(links = links, timeStep = timeStep, select = select, From 8318486966104bbde24e77c4b8aab7572b94e6a6 Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 12 Nov 2024 17:44:26 +0100 Subject: [PATCH 17/24] Split .importOutputForClusters modulation vs no modulation, lowerize columns, integrate endpoint mc-ind --- R/importOutput.R | 198 +++++++++++++++++++++++++++++------------------ R/zzz.R | 2 +- 2 files changed, 124 insertions(+), 76 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 99b6dc2c..75e01a77 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -340,6 +340,8 @@ cols_to_integer <- intersect(cols_to_integer, res_cols) res[,(cols_to_integer):=lapply(.SD, as.integer), .SDcols=cols_to_integer] + res[,(cols_to_factor):=lapply(.SD, tolower), .SDcols=cols_to_factor] + return(res) } @@ -421,6 +423,13 @@ } +#' Read output data for clusters if mustRun is disabled +#' +#' @return +#' a data.table +#' +#' @noRd +#' .importOutputForClusters_wo_mustrun <- function(areas, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel){ if (is_api_study(opts)) { @@ -446,11 +455,121 @@ } +#' Read output data for clusters if modulation is computed as inactive +#' +#' @return +#' a data.table +#' +#' @noRd +#' +.importOutputForClusters_wo_modulation <- function(areas, timeStep, select = NULL, mcYears = NULL, + showProgress, opts, reshapeFun, parallel) { + + # We should not \o/ + if (is_api_study(opts)) { + res <- .api_get_aggregate_areas(areas = areas, + timeStep = timeStep, + query_file = "details", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + res <- .format_api_aggregate_result(res) + } else { + res <- suppressWarnings( + suppressWarnings( + .importOutput("areas", "details", "area", areas, timeStep, NULL, + mcYears, showProgress, opts, reshapeFun, sameNames = FALSE, + objectDisplayName = "cluster", parallel = parallel) + ) + ) + } + + res[, mustRunPartial := 0L] +} + + +#' Read output data for clusters if modulation is computed as active +#' +#' @return +#' a data.table +#' +#' @noRd +#' +.importOutputForClusters_w_modulation <- function(areas, timeStep, select = NULL, mcYears = NULL, + showProgress, opts, reshapeFun, parallel, mod, clusterDesc) { + + if(timeStep!="hourly"){ + warning('Hourly data will be imported to compute partial must run min(production_t, capacity * minGenModulation_t). These data will be aggregated at the desired `timeStep`. ') + + #copy of warning in ChangeTimeStep + warning('Aggregation will be perform approximatively because optimization variables in ANTARES are doubles but ANTARES write only integers in TXT files, with this transformation we lose precision. If you want accurate data then you must import the corresponding data with `readAntares`') + + messageWarningMcYears<-paste0("When mcYears is set to all or NULL : ", mcYears, " and timeStep is set to : " ,timeStep , " result for mustRun are not accurate. Hourly `synthetic` or `details` results will be aggregated at the desired `timeStep`. " ) + + if( is.null(mcYears) ){ + warning(messageWarningMcYears, call. = FALSE) + }else if (is.character(mcYears)){ + if (mcYears=="all"){ + warning(messageWarningMcYears, call. = FALSE) + } + }else if (length(mcYears) > 1){ + warning(messageWarningMcYears, call. = FALSE) + } + } + + mod[is.na(minGenModulation), minGenModulation := 0] + + .mergeByRef(mod, clusterDesc) + mod[, mustRunPartial := minGenModulation * capacity] + + setkey(mod, area, cluster, timeId) + + if (is_api_study(opts)) { + res <- .api_get_aggregate_areas(areas = areas, + timeStep = "hourly", + query_file = "details", + select = select, + mcYears = mcYears, + synthesis = FALSE, + opts = opts + ) + res <- .format_api_aggregate_result(res) + mustRunPartial <- mod[J(res$area, res$cluster, res$timeId), mustRunPartial] + res[, mustRunPartial := pmin(production, mustRunPartial)] + res <- changeTimeStep(res, timeStep, "hourly", fun = "sum", opts = opts) + } else { + processFun <- function(x) { + x <- reshapeFun(x) + mustRunPartial <- mod[J(x$area, x$cluster, x$timeId), mustRunPartial] + x[, mustRunPartial := pmin(production, mustRunPartial)] + changeTimeStep(x, timeStep, "hourly", fun = "sum", opts = opts) + } + res <- suppressWarnings( + .importOutput("areas", "details", "area", areas, "hourly", NULL, + mcYears, showProgress, opts, processFun, + sameNames = FALSE, objectDisplayName = "cluster", + parallel = parallel) + + ) + + } + + return(res) +} + + +#' Read output data for clusters if mustRun is enabled +#' +#' @return +#' a data.table +#' +#' @noRd +#' .importOutputForClusters_w_mustrun <- function(areas, timeStep, select = NULL, mcYears = NULL, showProgress, opts, parallel) { - is_api <- is_api_study(opts) - reshapeFun <- function(x){ .reshape_details_file(x,file_type="details",opts=opts) } @@ -486,81 +605,10 @@ # Should we compute the partial must run ? if (is.null(mod$minGenModulation) || all(is.na(mod$minGenModulation) | mod$minGenModulation == 0)) { # We should not \o/ - if (is_api) { - res <- .api_get_aggregate_areas(areas = areas, - timeStep = timeStep, - query_file = "details", - select = select, - mcYears = mcYears, - synthesis = FALSE, - opts = opts - ) - res <- .format_api_aggregate_result(res) - } else { - res <- suppressWarnings( - suppressWarnings( - .importOutput("areas", "details", "area", areas, timeStep, NULL, - mcYears, showProgress, opts, reshapeFun, sameNames = FALSE, - objectDisplayName = "cluster", parallel = parallel) - ) - ) - } - res[, mustRunPartial := 0L] + res <- .importOutputForClusters_wo_modulation(areas, timeStep, select, mcYears, showProgress, opts, reshapeFun, parallel) } else { - # Worst case ! We have to ! - # - if(timeStep!="hourly"){ - warning('Hourly data will be imported to compute partial must run min(production_t, capacity * minGenModulation_t). These data will be aggregated at the desired `timeStep`. ') - - #copy of warning in ChangeTimeStep - warning('Aggregation will be perform approximatively because optimization variables in ANTARES are doubles but ANTARES write only integers in TXT files, with this transformation we lose precision. If you want accurate data then you must import the corresponding data with `readAntares`') - - messageWarningMcYears<-paste0("When mcYears is set to all or NULL : ", mcYears, " and timeStep is set to : " ,timeStep , " result for mustRun are not accurate. Hourly `synthetic` or `details` results will be aggregated at the desired `timeStep`. " ) - - if( is.null(mcYears) ){ - warning(messageWarningMcYears, call. = FALSE) - }else if (is.character(mcYears)){ - if (mcYears=="all"){ - warning(messageWarningMcYears, call. = FALSE) - } - }else if (length(mcYears) > 1){ - warning(messageWarningMcYears, call. = FALSE) - } - } - - mod[is.na(minGenModulation), minGenModulation := 0] - - .mergeByRef(mod, clusterDesc) - mod[, mustRunPartial := minGenModulation * capacity] - - setkey(mod, area, cluster, timeId) - - if (is_api) { - res <- .api_get_aggregate_areas(areas = areas, - timeStep = "hourly", - query_file = "details", - select = select, - mcYears = mcYears, - synthesis = FALSE, - opts = opts - ) - res <- .format_api_aggregate_result(res) - } else { - processFun <- function(x) { - x <- reshapeFun(x) - mustRunPartial <- mod[J(x$area, x$cluster, x$timeId), mustRunPartial] - x[, mustRunPartial := pmin(production, mustRunPartial)] - changeTimeStep(x, timeStep, "hourly", fun = "sum", opts = opts) - } - res <- suppressWarnings( - .importOutput("areas", "details", "area", areas, "hourly", NULL, - mcYears, showProgress, opts, processFun, - sameNames = FALSE, objectDisplayName = "cluster", - parallel = parallel) - ) - } - + res <- .importOutputForClusters_w_modulation(areas, timeStep, select, mcYears, showProgress, opts, reshapeFun, parallel, mod, clusterDesc) } .mergeByRef(res, clusterDesc[,.(area, cluster, `must-run`, `min-stable-power`)]) diff --git a/R/zzz.R b/R/zzz.R index 6c922c84..c8e5a6ce 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -110,7 +110,7 @@ utils::globalVariables( "Folder", "Mode", "Stats", "Name", "progNam", "mrgprice", "isLOLD_cum", "...To", "upstream", "downstream", "LOLD", "LOLD_data", "LOLP", "warn_for_status", "MRG. PRICE", "H. LEV", "V2", "V1", "size", "ORDINAL_POSITION_BY_TOPIC", - "DETAILS_FILES_TYPE","ANTARES_DISPLAYED_NAME") + "DETAILS_FILES_TYPE","ANTARES_DISPLAYED_NAME", "time") ) ## INPUT Properties REF ---- From 09d9e331f30be6bb1177bd24f48ba121a5ae1ed1 Mon Sep 17 00:00:00 2001 From: kemihak Date: Tue, 12 Nov 2024 21:40:56 +0100 Subject: [PATCH 18/24] columns to convert to factor are the same to convert to lower case --- R/importOutput.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 75e01a77..84a188c0 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -332,16 +332,15 @@ res[,time:=NULL] } - cols_to_factor <- c("area", "link", "cluster") - cols_to_factor <- intersect(cols_to_factor, res_cols) - res[,(cols_to_factor):=lapply(.SD, as.factor), .SDcols=cols_to_factor] + cols_to_factor_lower <- c("area", "link", "cluster") + cols_to_factor_lower <- intersect(cols_to_factor_lower, res_cols) + res[,(cols_to_factor_lower):=lapply(.SD, as.factor), .SDcols=cols_to_factor_lower] + res[,(cols_to_factor_lower):=lapply(.SD, tolower), .SDcols=cols_to_factor_lower] cols_to_integer <- c("timeId") cols_to_integer <- intersect(cols_to_integer, res_cols) res[,(cols_to_integer):=lapply(.SD, as.integer), .SDcols=cols_to_integer] - res[,(cols_to_factor):=lapply(.SD, tolower), .SDcols=cols_to_factor] - return(res) } @@ -540,12 +539,14 @@ res[, mustRunPartial := pmin(production, mustRunPartial)] res <- changeTimeStep(res, timeStep, "hourly", fun = "sum", opts = opts) } else { + processFun <- function(x) { x <- reshapeFun(x) mustRunPartial <- mod[J(x$area, x$cluster, x$timeId), mustRunPartial] x[, mustRunPartial := pmin(production, mustRunPartial)] changeTimeStep(x, timeStep, "hourly", fun = "sum", opts = opts) } + res <- suppressWarnings( .importOutput("areas", "details", "area", areas, "hourly", NULL, mcYears, showProgress, opts, processFun, @@ -553,7 +554,6 @@ parallel = parallel) ) - } return(res) From 8f504182d997ccdf98b0665eb9ebd7d311bb2bb2 Mon Sep 17 00:00:00 2001 From: kemihak Date: Wed, 13 Nov 2024 17:06:43 +0100 Subject: [PATCH 19/24] Add comment --- R/importOutput.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/importOutput.R b/R/importOutput.R index 84a188c0..eb4d1ea0 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -335,8 +335,10 @@ cols_to_factor_lower <- c("area", "link", "cluster") cols_to_factor_lower <- intersect(cols_to_factor_lower, res_cols) res[,(cols_to_factor_lower):=lapply(.SD, as.factor), .SDcols=cols_to_factor_lower] + + #To remove when endpoint will not send uppercase anymore res[,(cols_to_factor_lower):=lapply(.SD, tolower), .SDcols=cols_to_factor_lower] - + cols_to_integer <- c("timeId") cols_to_integer <- intersect(cols_to_integer, res_cols) res[,(cols_to_integer):=lapply(.SD, as.integer), .SDcols=cols_to_integer] From 269d5118d808327d32d128467150ab9838d97826 Mon Sep 17 00:00:00 2001 From: kemihak Date: Wed, 13 Nov 2024 18:07:50 +0100 Subject: [PATCH 20/24] tolower then factor, order is important to prevent char conversion --- R/importOutput.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index eb4d1ea0..759107fd 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -334,10 +334,9 @@ cols_to_factor_lower <- c("area", "link", "cluster") cols_to_factor_lower <- intersect(cols_to_factor_lower, res_cols) - res[,(cols_to_factor_lower):=lapply(.SD, as.factor), .SDcols=cols_to_factor_lower] - #To remove when endpoint will not send uppercase anymore res[,(cols_to_factor_lower):=lapply(.SD, tolower), .SDcols=cols_to_factor_lower] + res[,(cols_to_factor_lower):=lapply(.SD, as.factor), .SDcols=cols_to_factor_lower] cols_to_integer <- c("timeId") cols_to_integer <- intersect(cols_to_integer, res_cols) From 379b1960f1598d8a699b7d4ffbbd94ff135b309b Mon Sep 17 00:00:00 2001 From: kemihak Date: Wed, 13 Nov 2024 21:40:00 +0100 Subject: [PATCH 21/24] Add control on clustersRes before setting areas to "all" --- R/readAntares.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/readAntares.R b/R/readAntares.R index 11f75659..a6ea598a 100644 --- a/R/readAntares.R +++ b/R/readAntares.R @@ -911,7 +911,7 @@ readAntaresAreas <- function(areas, links = TRUE, clusters = TRUE, clustersRes = if ("mcYears" %in% unlist(select) & is.null(mcYears)) mcYears <- "all" # If all arguments are NULL, import all areas - if (is.null(areas) & is.null(links) & is.null(clusters) & is.null(districts) & is.null(clustersST)) { + if (is.null(areas) & is.null(links) & is.null(clusters) & is.null(districts) & is.null(clustersRes) & is.null(clustersST)) { areas <- "all" } From 74e23111ffba9bc7aad72cafb968c8228d5c2e29 Mon Sep 17 00:00:00 2001 From: kemihak Date: Thu, 14 Nov 2024 06:35:04 +0100 Subject: [PATCH 22/24] REemove additional attributes to match the legacy ones --- R/importOutput.R | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 759107fd..6d41e83e 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -314,10 +314,11 @@ } endpoint <- paste0(endpoint_root, query_file_url, frequency_url, columns_url, areas_url, mc_years_url) - res <- api_get(opts = opts, - endpoint = endpoint, - default_endpoint = "v1/studies") - + res <- api_get(opts = opts, endpoint = endpoint, default_endpoint = "v1/studies") + + attr(res, "spec") <- NULL + attr(res, "problems") <- NULL + return(as.data.table(res)) } From 0067641239df75e049c5860bbf65b0af834624bc Mon Sep 17 00:00:00 2001 From: kemihak Date: Thu, 14 Nov 2024 15:51:30 +0100 Subject: [PATCH 23/24] Remove additional attributes in link part to match the legacy ones --- R/importOutput.R | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 6d41e83e..38612ff4 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -985,9 +985,10 @@ } endpoint <- paste0(endpoint_root, frequency_url, columns_url, links_url, mc_years_url) - res <- api_get(opts = opts, - endpoint = endpoint, - default_endpoint = "v1/studies") + res <- api_get(opts = opts, endpoint = endpoint, default_endpoint = "v1/studies") + + attr(res, "spec") <- NULL + attr(res, "problems") <- NULL return(as.data.table(res)) } From 01a17f582fbd0fc487b1ca055b0d72c75de9d270 Mon Sep 17 00:00:00 2001 From: kemihak Date: Thu, 12 Dec 2024 11:11:25 +0100 Subject: [PATCH 24/24] Endpoint does not send time column anymore --- R/importOutput.R | 4 ---- 1 file changed, 4 deletions(-) diff --git a/R/importOutput.R b/R/importOutput.R index 38612ff4..a675ecc9 100644 --- a/R/importOutput.R +++ b/R/importOutput.R @@ -328,10 +328,6 @@ if (is.null(res)) return(NULL) res_cols <- colnames(res) - #To remove when endpoint will not send time anymore - if ("time" %in% res_cols) { - res[,time:=NULL] - } cols_to_factor_lower <- c("area", "link", "cluster") cols_to_factor_lower <- intersect(cols_to_factor_lower, res_cols)