From 532710e970692ad9fdc3374a73b2419ec920df12 Mon Sep 17 00:00:00 2001 From: Pakillo Date: Sat, 17 Mar 2018 18:35:59 +0100 Subject: [PATCH 1/4] delete old files --- R/get_aemet_mensual.R | 223 ------------------------------------ R/get_aemet_normalized.R | 61 ---------- man/get_aemet_normalized.Rd | 25 ---- 3 files changed, 309 deletions(-) delete mode 100644 R/get_aemet_mensual.R delete mode 100644 R/get_aemet_normalized.R delete mode 100644 man/get_aemet_normalized.Rd diff --git a/R/get_aemet_mensual.R b/R/get_aemet_mensual.R deleted file mode 100644 index 630b193..0000000 --- a/R/get_aemet_mensual.R +++ /dev/null @@ -1,223 +0,0 @@ -#' Download monthly climatic data from AEMET -#' -#' @param api -#' @param dates -#' @param station_id -#' @param export -#' @param exportDir -#' @param exportFormat -#' @param metadatafile -#' @param verbose -#' -#' @return -#' -#' @importFrom meteoland downloadAEMETstationdata -#' - -# aemet_mensual <- function(api, dates, station_id, export = FALSE, -# exportDir = getwd(), -# exportFormat = "meteoland", -# metadatafile = "MP.txt", verbose=TRUE){ -# # dates must be a year (or vector of years) "yyyy" -# # station_id is the id code of the wanted AEMET station -# -# # set curl options -# h = new_handle() -# handle_setheaders(h, "Cache-Control" = "no-cache", api_key = api) -# handle_setopt(h, ssl_verifypeer = FALSE) -# handle_setopt(h, customrequest = "GET") -# -# -# ## Download station metadata -# spdflist = meteoland::downloadAEMETstationdata(api, station_id) -# -# if (sum(station_id %in% rownames(spdflist@data)) != length(station_id)) { -# warning(paste("Could not find the following station(s):\n", -# paste(station_id[!station_id %in% rownames(spdflist@data)], -# collapse = ", "), ".\n", sep = ""), -# immediate. = TRUE) -# } -# -# station_id = station_id[station_id %in% rownames(spdflist@data)] -# spdflist = spdflist[station_id,] -# points = as(spdflist,"SpatialPoints") -# dfout = spdflist@data -# -# npoints = length(station_id) -# nyears = length(dates) -# -# ## Output data frame meta data -# dfout$dir = as.character(rep(exportDir, npoints)) -# dfout$filename = as.character(paste(station_id,".txt",sep = "")) -# rownames(dfout) = station_id -# -# stationfound = rep(FALSE, length(station_id)) -# -# ## Get initial and final years -# intlength = 1 -# steps_ini <- seq(1, length(dates), by = intlength) -# steps_fin <- pmin(steps_ini + intlength - 1, length(dates)) # does not allow holes in the date sequence -# dates_seq <- list(ini = dates[steps_ini], fin = dates[steps_fin]) -# -# -# # express dates as url -# url_header <- "https://opendata.aemet.es/opendata/api/valores/climatologicos/mensualesanuales/datos/" -# station_url <- rep(station_id, each = length(steps_ini)) -# url <- matrix( -# paste(url_header, "anioini/", dates_seq$ini, "/aniofin/", dates_seq$fin, "/estacion/", station_url, sep = ""), -# ncol = length(station_id), dimnames = list(NULL, station_id)) -# -# -# -# -# -# -# -# -# -# # send request and format the output -# data_list <- vector("list", npoints) -# names(data_list) <- colnames(url) -# -# -# for (j in 1:ncol(url)) { -# if (verbose) cat(paste("\n Downloading data for station '", station_id[j],"' (", j, "/", npoints, ")\n", sep = "")) -# -# -# -# # dfread <- matrix(NA, ncol = 7, nrow = 0, dimnames = list(NULL, c("date", "MeanTemperature", "Precipitation", -# # "MinTemperature", "MaxTemperature", "WindDirection", "WindSpeed"))) -# -# -# -# t0 <- c() -# t1 <- c() -# t2 <- c() -# if (verbose) pb = txtProgressBar(0, max = nrow(url), style = 3) -# -# -# -# -# -# -# -# -# for (i in 1:nrow(url)) { -# -# if (verbose) setTxtProgressBar(pb, i) -# t0[i] <- Sys.time() -# -# data_raw <- curl_fetch_memory(url[i,j], h) -# urldatos <- strsplit(rawToChar(data_raw$content), split = "\"")[[1]][10] -# urlmetadatos <- strsplit(rawToChar(md_raw$content), split = "\"")[[1]][14] -# #data_raw <- curl_fetch_memory(urldatos, h) # from meteoland -# -# url2 <- httr::handle(urldatos) -# httr::set_config(httr::config(ssl_verifypeer = 0L)) -# datos <- httr::GET(handle = url2) -# data.json <- httr::content(datos, type = "text/plain", encoding = "ISO-8859-15") -# library(RJSONIO) -# datalist <- fromJSON(data.json) -# datalist <- datalist[-13] # last element are annual data -# dataf <- lapply(datalist, function(x) {as.data.frame(t(x))}) -# # now bind all data frames (considering variables!) -# -# t1[i] <- Sys.time() -# -# -# -# -# -# -# -# -# -# -# -# -# -# -# -# data_raw <- data_raw$content[-which(data_raw$content %in% c("5b", "20", "7b", "20", "20", "22", "5d"))] -# data_raw <- strsplit(rawToChar(data_raw), split = "}") -# data_raw <- strsplit(data_raw[[1]], split = "\n") -# -# data_names <- lapply(data_raw, FUN = function(x){strsplit(x, split = ":")}) -# data_names <- lapply(data_names, FUN = function(x){unlist(lapply(x, FUN = function(x){x[1]}))}) -# data_names_val <- lapply(data_names, FUN = function(x){v <- which(x %in% c("fecha", "tmed", "prec", "tmin", "tmax", "dir", "velmedia")) ; names(v) <- x[v] ; return(v)}) -# data_names_info <- lapply(data_names, FUN = function(x){which(x %in% c("indicativo", "nombre", "provincia", "altitud"))})[[1]] -# -# # data_info <- unlist(lapply(data_raw, FUN = function(x){strsplit(x[data_names_info], split = ":")})[[1]]) # UNUSED for now -# # data_info <- matrix(sub(data_info, pattern = ",", replacement = ""), ncol = 2, dimnames = list(NULL, c("attribute", "value")), byrow = T) -# -# data_value <- list() -# for(k in 1:length(data_names_val)){ -# data_value[[k]] <- strsplit(data_raw[[k]][data_names_val[[k]][c("fecha", "tmed", "prec", "tmin", "tmax", "dir", "velmedia")]], split = ":") -# } -# data_value <- unlist(lapply(data_value, FUN = function(x){lapply(x, FUN = function(x){x[length(x)]})})) -# data_value <- unlist(lapply(data_value, FUN = function(x){ -# if(!is.na(x)){ -# if(substr(x, start = nchar(x), stop = nchar(x)) == ","){ -# r <- substr(x, start = 1, stop = nchar(x)-1) -# return(r) -# }else(return(x)) -# }else(return(NA)) -# })) -# data_value <- sub(data_value, pattern = ",", replacement = ".") -# data_value <- sub(data_value, pattern = "Ip", replacement = "NA") -# data_value <- matrix(data_value, ncol = 7, byrow = T) -# -# dfread <- rbind(dfread, data_value) -# t2[i] <- Sys.time() -# } -# -# # print(paste("Station ", colnames(url)[j],": downloading time = ", round(sum(t1-t0)/sum(t2-t0)*100), "% of total", sep ="")) -# -# df <- data.frame( -# DOY =as.POSIXlt(dates)$yday, -# MeanTemperature=rep(NA, ndays), -# MinTemperature=rep(NA, ndays), -# MaxTemperature=rep(NA, ndays), -# Precipitation=rep(NA, ndays), -# MeanRelativeHumidity=rep(NA, ndays), -# MinRelativeHumidity=rep(NA, ndays), -# MaxRelativeHumidity=rep(NA, ndays), -# Radiation=rep(NA, ndays), -# WindSpeed=rep(NA, ndays), -# WindDirection=rep(NA, ndays)) -# rownames(df) = as.character(dates) -# -# if(sum(!is.na(dfread[,"date"]))>0) { -# stationfound[j] = TRUE -# dfread = dfread[!is.na(dfread[,"date"]),] -# data_date <- dfread[,"date"] -# df[data_date,"MeanTemperature"] <- as.numeric(dfread[,"MeanTemperature"]) -# df[data_date,"MinTemperature"] <- as.numeric(dfread[,"MinTemperature"]) -# df[data_date,"MaxTemperature"] <- as.numeric(dfread[,"MaxTemperature"]) -# df[data_date,"Precipitation"] <- as.numeric(dfread[,"Precipitation"]) -# df[data_date,"WindDirection"] <- as.numeric(dfread[,"WindDirection"]) -# df[data_date,"WindSpeed"] <- as.numeric(dfread[,"WindSpeed"]) -# # Export the data -# if(export){ -# if(dfout$dir[j]!="") f = paste(dfout$dir[j],dfout$filename[j], sep="/") -# else f = dfout$filename[j] -# writemeteorologypoint(df, f, exportFormat) -# if(verbose) cat(paste("\n File written to ",f, "\n", sep="")) -# if(exportDir!="") f = paste(exportDir,metadatafile, sep="/") -# else f = metadatafile -# spdf = SpatialPointsDataFrame(points, dfout) -# write.table(as.data.frame(spdf),file= f,sep="\t", quote=FALSE) -# } else { -# data_list[[j]] = df -# } -# } else { -# stationfound[j] = FALSE -# } -# } -# data_list = data_list[stationfound] -# -# if(length(stationfound)>0) warning(paste("\nInformation could not be retrieved for the following stations: \n ", -# paste(station_id[stationfound], collapse=", "), ".\n", sep="")) -# if(!export) return(SpatialPointsMeteorology(points = points, data = data_list, dates = dates)) -# invisible(spdf) -# } diff --git a/R/get_aemet_normalized.R b/R/get_aemet_normalized.R deleted file mode 100644 index 71ac8b1..0000000 --- a/R/get_aemet_normalized.R +++ /dev/null @@ -1,61 +0,0 @@ - -#' Download normalized data from a list of AEMET stations -#' -#' @param apikey String with the AEMET API key (see https://opendata.aemet.es/). -#' @param station_id Vector with station_id (see \code{\link{aemet_stations}}). -#' -#' @return A data frame. -#' @export -#' -#' @import httr -#' -#' @examples \dontrun{ -#' grazalema <- get_aemet_normalized(apikey, "5911A") -#' } -#' -get_aemet_normalized <- function(apikey, station_id) { - - if (length(station_id) < 2) { - dat <- .download_aemet_normalized(apikey, station_id) - } else{ - dat <- tryCatch({.download_aemet_normalized(apikey, station_id[1])}) - c <- 0 - for (i in 2:length(station_id)) { - c <- c+1 - print(station_id[i]) - if (c %% 10 == 0) Sys.sleep(10) - temp <- .download_aemet_normalized(apikey, station_id[i]) - dat <- rbind(dat, temp) - } - } - dat -} - - - - - - -.download_aemet_normalized <- function(apikey, station_id) { - - h = new_handle() - handle_setheaders(h, `Cache-Control` = "no-cache", api_key = apikey) - handle_setopt(h, ssl_verifypeer = FALSE) - handle_setopt(h, customrequest = "GET") - url <- paste("https://opendata.aemet.es/opendata/api/valores/climatologicos/normales/estacion/", paste(station_id, collapse = ","), sep = "") - md_raw <- curl_fetch_memory(url, h) - urldatos <- strsplit(rawToChar(md_raw$content), split = "\"")[[1]][10] - - url2 <- httr::handle(urldatos) - set_config(config(ssl_verifypeer = 0L)) - q1 <- httr::GET(handle = url2) - dataQ1 <- httr::content(q1, type = "text/plain", encoding = "ISO-8859-15") - listQ1 <- fromJSON(dataQ1) - df <- t(sapply(listQ1, function(e) e)) - station_id <- df[, "indicativo"] - df2 <- apply(df, 2, as.character) - df3 <- data.frame(apply(df2, 2, as.numeric)) - df3$indicativo <- station_id - colnames(df3)[1] <- "station_id" - return(df3) -} diff --git a/man/get_aemet_normalized.Rd b/man/get_aemet_normalized.Rd deleted file mode 100644 index acec5b2..0000000 --- a/man/get_aemet_normalized.Rd +++ /dev/null @@ -1,25 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/get_aemet_normalized.R -\name{get_aemet_normalized} -\alias{get_aemet_normalized} -\title{Download normalized data from a list of AEMET stations} -\usage{ -get_aemet_normalized(apikey, station_id) -} -\arguments{ -\item{apikey}{String with the AEMET API key (see https://opendata.aemet.es/).} - -\item{station_id}{Vector with station_id (see \code{\link{aemet_stations}}).} -} -\value{ -A data frame. -} -\description{ -Download normalized data from a list of AEMET stations -} -\examples{ -\dontrun{ -grazalema <- get_aemet_normalized(apikey, "5911A") -} - -} From 1ac931479fe509d46c4b22928db5e8cb00894759 Mon Sep 17 00:00:00 2001 From: Pakillo Date: Sat, 17 Mar 2018 18:36:49 +0100 Subject: [PATCH 2/4] parsing of urls now based on regular expressions TO DO: switch from stringr to base R --- R/get_data.R | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/R/get_data.R b/R/get_data.R index ddfe796..4944f0e 100644 --- a/R/get_data.R +++ b/R/get_data.R @@ -7,6 +7,7 @@ #' @export #' @import httr #' @importFrom jsonlite fromJSON +#' @importFrom stringr str_extract_all #' #' get_data <- function(apidest, apikey) { @@ -17,8 +18,14 @@ get_data <- function(apidest, apikey) { path1 <- httr::GET(url1, add_headers(api_key = apikey)) - url.data <- httr::content(path1, as = "parsed")$datos - url.metadata <- httr::content(path1, as = "parsed")$metadatos + urls.text <- httr::content(path1, as = "text") + + ## TO DO: use base R rather than stringr + ## Also extract urls more safely + urls <- unlist(stringr::str_extract_all(urls.text, + pattern = "https://opendata.aemet.es/opendata/sh/([:alnum:])+")) + url.data <- urls[1] + url.metadata <- urls[2] #path2 <- GET(url.data, add_headers(api_key = apikey)) # it seems apikey not necessary for this step path2 <- httr::GET(url.data) @@ -27,4 +34,4 @@ get_data <- function(apidest, apikey) { datos -} \ No newline at end of file +} From 7f8aa0f13d9dd85f7ddab1e9aaa732ea278211cf Mon Sep 17 00:00:00 2001 From: Pakillo Date: Sat, 17 Mar 2018 18:37:22 +0100 Subject: [PATCH 3/4] add function to retrieve station normal climatology 1981-2010 --- DESCRIPTION | 6 ++++-- NAMESPACE | 3 ++- R/aemet_climatology_station.R | 20 ++++++++++++++++++++ man/aemet_climatology_station.Rd | 24 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 R/aemet_climatology_station.R create mode 100644 man/aemet_climatology_station.Rd diff --git a/DESCRIPTION b/DESCRIPTION index a0e6fde..37d097c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -3,8 +3,9 @@ Type: Package Title: Obtain Climatic and Meteorological Data from Spanish Meteorological Agency (AEMET) Version: 0.1.2 Date: 2018-03-17 -Authors@R: c(person("Francisco", "Balao", role = "aut"), +Authors@R: c( person("Francisco", "Rodriguez-Sanchez", email = "f.rodriguez.sanc@gmail.com", role = c("aut", "cre")), + person("Francisco", "Balao", role = "aut"), person("Francisco", "Viciana", role = "aut"), person("Pedro", "Molina", role = "aut"), person("Jose", "Alcaide", role = "aut"), @@ -20,7 +21,8 @@ Depends: R (>= 3.0.0) Imports: httr, - jsonlite + jsonlite, + stringr VignetteBuilder: knitr RoxygenNote: 6.0.1 diff --git a/NAMESPACE b/NAMESPACE index 9946e5e..d41e7b2 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,8 +1,9 @@ # Generated by roxygen2: do not edit by hand +export(aemet_climatology_station) export(aemet_stations) export(dms2decdegrees) -export(get_aemet_normalized) export(get_data) import(httr) importFrom(jsonlite,fromJSON) +importFrom(stringr,str_extract_all) diff --git a/R/aemet_climatology_station.R b/R/aemet_climatology_station.R new file mode 100644 index 0000000..ee9a279 --- /dev/null +++ b/R/aemet_climatology_station.R @@ -0,0 +1,20 @@ +#' Get normal climatology values for a station +#' +#' Standard climatology from 1981 to 2010. +#' +#' @param station Station identifier code (see \code{\link{aemet_stations}}). +#' @param apikey Personal API key (see \url{https://opendata.aemet.es/centrodedescargas/inicio}). +#' +#' @return a data.frame +#' @export +#' +#' @examples \dontrun{ +#' aemet_climatology_station("5911A") +#' } +aemet_climatology_station <- function(station, apikey) { + + apidest <- paste0("/api/valores/climatologicos/normales/estacion/", station) + + clim <- get_data(apidest, apikey) + +} \ No newline at end of file diff --git a/man/aemet_climatology_station.Rd b/man/aemet_climatology_station.Rd new file mode 100644 index 0000000..5750e3b --- /dev/null +++ b/man/aemet_climatology_station.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/aemet_climatology_station.R +\name{aemet_climatology_station} +\alias{aemet_climatology_station} +\title{Get normal climatology values for a station} +\usage{ +aemet_climatology_station(station, apikey) +} +\arguments{ +\item{station}{Station identifier code (see \code{\link{aemet_stations}}).} + +\item{apikey}{Personal API key (see \url{https://opendata.aemet.es/centrodedescargas/inicio}).} +} +\value{ +a data.frame +} +\description{ +Standard climatology from 1981 to 2010. +} +\examples{ +\dontrun{ +aemet_climatology_station("5911A") +} +} From a5c91981bb2e97971cdff8380d2e27c97b47b57d Mon Sep 17 00:00:00 2001 From: Pakillo Date: Sat, 17 Mar 2018 18:45:12 +0100 Subject: [PATCH 4/4] update readme --- README.Rmd | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.Rmd b/README.Rmd index 97f9d26..528e481 100644 --- a/README.Rmd +++ b/README.Rmd @@ -31,6 +31,6 @@ To be able to download data from AEMET you will need a free API key which you ca ```{r eval=FALSE} library(aemet) stations <- aemet_stations(apikey) -grazalema <- get_aemet_normalized(apikey, "5911A") +grazalema <- aemet_climatology_station("5911A", apikey) ``` diff --git a/README.md b/README.md index 5b9f48c..4d013c9 100644 --- a/README.md +++ b/README.md @@ -23,5 +23,5 @@ Usage ``` r library(aemet) stations <- aemet_stations(apikey) -grazalema <- get_aemet_normalized(apikey, "5911A") +grazalema <- aemet_climatology_station("5911A", apikey) ```