From 4d788a3009141732705af9eb37feeb66db99f023 Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Mon, 7 Oct 2024 13:29:07 -0400 Subject: [PATCH 1/4] fix double URL; directory substructure --- R/download.R | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/R/download.R b/R/download.R index c1c741d..3d39a7b 100644 --- a/R/download.R +++ b/R/download.R @@ -2517,7 +2517,7 @@ download_modis <- function( ) message("Requests were processed.\n") - return(NULL) + return(download_hash(hash, directory_to_save)) } @@ -2591,27 +2591,44 @@ download_modis <- function( rvest::html_elements("tr") |> rvest::html_attr("data-path") - filelist_sub <- + filelist_sub <- grep( paste0("(", paste(tiles_requested, collapse = "|"), ")"), filelist, value = TRUE ) - download_url <- sprintf("%s%s", ladsurl, filelist_sub) download_name <- sapply( - strsplit(download_url, paste0("/", day, "/")), `[`, 2 + strsplit(filelist_sub, paste0("/", day, "/")), `[`, 2 + ) + + dir_str_julian <- + lapply(download_name, function(x) strsplit(x, ".A")[[1]][3]) + + dir_substr <- paste0( + substr(dir_str_julian, 1, 4), "/", + substr(dir_str_julian, 5, 7), "/" + ) + + new_dirs <- unique( + sprintf("%s%s", directory_to_save, dir_substr) + ) + + lapply( + new_dirs, + function(x) dir.create(x, recursive = TRUE, showWarnings = FALSE) ) # Main wget run download_command <- paste0( "wget -e robots=off -np -R .html,.tmp ", "-nH --cut-dirs=3 \"", - download_url, + filelist_sub, "\" --header \"Authorization: Bearer ", nasa_earth_data_token, "\" -O ", directory_to_save, + dir_substr, download_name, "\n" ) @@ -2620,7 +2637,7 @@ download_modis <- function( download_command <- download_command[ which( !file.exists( - paste0(directory_to_save, download_name) + paste0(directory_to_save, dir_substr, download_name) ) ) ] From acd3fa11a6df6035948b50d9b0b772d5eae28de5 Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Mon, 7 Oct 2024 14:23:01 -0400 Subject: [PATCH 2/4] MOD06_L2; document --- R/download.R | 14 ++++++++++++-- man/download_modis.Rd | 2 +- tests/testthat/test-terraclimate.R | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/R/download.R b/R/download.R index 3d39a7b..0ba3b75 100644 --- a/R/download.R +++ b/R/download.R @@ -2279,7 +2279,7 @@ download_koppen_geiger <- function( #' @return #' * For \code{hash = FALSE}, NULL #' * For \code{hash = TRUE}, an \code{rlang::hash_file} character. -#' * HDF (.hdf) files will be stored in +#' * HDF (.hdf) files will be stored in year/day_of_year sub-directories within #' \code{directory_to_save}. #' @importFrom Rdpack reprompt #' @references @@ -2493,6 +2493,7 @@ download_modis <- function( nasa_earth_data_token, "\" -O ", directory_to_save, + splitter, download_name, "\n" ) @@ -2500,10 +2501,19 @@ download_modis <- function( #### filter commands to non-existing files download_command <- download_command[ which( - !file.exists(paste0(directory_to_save, download_name)) + !file.exists(paste0(directory_to_save, splitter, download_name)) ) ] + new_dirs <- unique( + sprintf("%s%s", directory_to_save, splitter) + ) + + lapply( + new_dirs, + function(x) dir.create(x, recursive = TRUE, showWarnings = FALSE) + ) + # avoid any possible errors by removing existing command files download_sink(commands_txt) #### cat command only if file does not already exist diff --git a/man/download_modis.Rd b/man/download_modis.Rd index 9e92cec..6c5945e 100644 --- a/man/download_modis.Rd +++ b/man/download_modis.Rd @@ -61,7 +61,7 @@ downloaded files. Default is \code{FALSE}.} \itemize{ \item For \code{hash = FALSE}, NULL \item For \code{hash = TRUE}, an \code{rlang::hash_file} character. -\item HDF (.hdf) files will be stored in +\item HDF (.hdf) files will be stored in year/day_of_year sub-directories within \code{directory_to_save}. } } diff --git a/tests/testthat/test-terraclimate.R b/tests/testthat/test-terraclimate.R index 2c6a9f5..ed42fe1 100644 --- a/tests/testthat/test-terraclimate.R +++ b/tests/testthat/test-terraclimate.R @@ -44,7 +44,7 @@ testthat::test_that("download_terraclimate (single year)", { # function parameters year <- 2019 variables <- "Precipitation" - directory_to_save <- paste0(tempdir(), "/terracclimate/") + directory_to_save <- paste0(tempdir(), "/terraclimate/") # run download function download_data(dataset_name = "terraclimate", year = year, From 13076f3c2bc1d3af73710f99ecb7492940766715 Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Mon, 7 Oct 2024 14:24:02 -0400 Subject: [PATCH 3/4] DESCRIPTION --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index a049e3a..b848e80 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: amadeus Title: Accessing and Analyzing Large-Scale Environmental Data -Version: 1.1.0 +Version: 1.1.1 Authors@R: c( person(given = "Mitchell", family = "Manware", role = c("aut", "ctb"), comment = c(ORCID = "0009-0003-6440-6106")), person(given = "Insang", family = "Song", role = c("aut", "ctb"), comment = c(ORCID = "0000-0001-8732-3256")), From d524c57f98b530004578d026ebb0d41cc541e757 Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Mon, 7 Oct 2024 14:42:49 -0400 Subject: [PATCH 4/4] lint --- R/download.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/download.R b/R/download.R index 0ba3b75..de5a871 100644 --- a/R/download.R +++ b/R/download.R @@ -2601,7 +2601,7 @@ download_modis <- function( rvest::html_elements("tr") |> rvest::html_attr("data-path") - filelist_sub <- + filelist_sub <- grep( paste0("(", paste(tiles_requested, collapse = "|"), ")"), filelist,