Merge pull request #111 from NIEHS/cran-0725

1.0.0
NIEHS · Jul 28, 2024 · f98e7d7 · f98e7d7
2 parents 6c51d42 + 433b099
commit f98e7d7
Show file tree

Hide file tree

Showing 25 changed files with 659 additions and 640 deletions.
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -1,4 +1,3 @@
-^vignettes/
 ^input/
 ^tools/
 ^manuscript/
@@ -7,7 +6,6 @@
 ^amadeus\.Rproj$
 ^\.Rproj\.user$
 ^\.github$
-^LICENSE\.md$
 ^_pkgdown\.yml$
 ^docs$
 ^pkgdown$
@@ -19,4 +17,6 @@ pkgdown
 output
 manuscript
 input
-tests
+tests
+vignettes/epa_download.Rmd
+R/olm_functions.R
diff --git a/.gitignore b/.gitignore
@@ -21,10 +21,6 @@
 # RStudio files
 .Rproj.user/
 
-# produced vignettes
-vignettes/*.html
-vignettes/*.pdf
-
 # OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3
 .httr-oauth
 
@@ -41,6 +37,7 @@ vignettes/*.pdf
 
 # pkgdown site
 docs/
+doc/
 
 # translation temp files
 po/*~
@@ -54,3 +51,6 @@ rsconnect/
 
 # custom linter dev
 \.github/linters
+/doc/
+/Meta/
+inst/doc
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -9,10 +9,10 @@ Authors@R: c(
     person("Kyle", "Messier", role = c("aut", "cre"), email = "[email protected]", comment = c(ORCID = "0000-0001-9508-9623"))
     )
 Maintainer: Kyle Messier <[email protected]>
-Description: amadeus is designed to faciliate access to and utiltiy with large scale, publicly available environmental data in R. The package contains funtions for downloading (download_data()) raw data files from web URLs, processing (process_covariates()) the raw data files into interpretable spatial objects, and extracting values (calc_covariates()) from the spatial data objects at point and polygon locations. These functions call a series of source-specific functions which are tailored to each data sources/datasets particular URL structure, data format, and spatio-temporal resolution. amadeus functions are tested, versioned, and open source and open access.
+Description: Functions are designed to facilitate access to and utility with large scale, publicly available environmental data in R. The package contains functions for downloading (download_data()) raw data files from web URLs, processing (process_covariates()) the raw data files into clean spatial objects, and extracting values (calc_covariates()) from the spatial data objects at point and polygon locations. These functions call a series of source-specific functions which are tailored to each data sources/datasets particular URL structure, data format, and spatial/temporal resolution. The functions are tested, versioned, and open source and open access.
 Depends: R (>= 4.1.0)
-Imports: dplyr, sf, sftime, stats, terra, methods, data.table, httr, rvest, exactextractr, utils, stringi, testthat (>= 3.0.0), parallelly, stars, future, future.apply, tidyr, rlang, rstac, nhdplusTools, archive, collapse, Rdpack
-Suggests: covr, withr, knitr, rmarkdown, lwgeom, FNN, doRNG
+Imports: dplyr, sf, sftime, stats, terra, methods, data.table, httr, rvest, exactextractr, utils, stringi, testthat (>= 3.0.0), parallelly, stars, future, future.apply, tidyr, rlang, nhdplusTools, archive, collapse, Rdpack
+Suggests: covr, withr, knitr, rmarkdown, lwgeom, FNN, doRNG, devtools, stringr
 RdMacros: Rdpack
 Encoding: UTF-8
 VignetteBuilder: knitr, rmarkdown

diff --git a/NAMESPACE b/NAMESPACE
@@ -50,7 +50,6 @@ export(download_modis)
 export(download_narr)
 export(download_nei)
 export(download_nlcd)
-export(download_olm)
 export(download_permit)
 export(download_prism)
 export(download_remove_command)
@@ -69,7 +68,6 @@ export(extract_urls)
 export(generate_date_sequence)
 export(generate_time_sequence)
 export(is_date_proper)
-export(list_stac_files)
 export(narr_variable)
 export(process_aqs)
 export(process_blackmarble)
@@ -99,7 +97,6 @@ export(process_modis_warp)
 export(process_narr)
 export(process_nei)
 export(process_nlcd)
-export(process_olm)
 export(process_prism)
 export(process_sedac_codes)
 export(process_sedac_groads)
@@ -161,11 +158,6 @@ importFrom(nhdplusTools,get_huc)
 importFrom(parallelly,availableWorkers)
 importFrom(rlang,inject)
 importFrom(rlang,sym)
-importFrom(rstac,assets_url)
-importFrom(rstac,links)
-importFrom(rstac,read_collections)
-importFrom(rstac,read_items)
-importFrom(rstac,read_stac)
 importFrom(sf,st_as_sf)
 importFrom(sf,st_as_sfc)
 importFrom(sf,st_bbox)

diff --git a/R/download.R b/R/download.R
@@ -34,6 +34,9 @@
 #' * \code{\link{download_nei}}: `"nei"`, `"NEI"`
 #' * \code{\link{download_gridmet}}: `"gridMET"`, `"gridmet"`
 #' * \code{\link{download_terraclimate}}: `"TerraClimate"`, `"terraclimate"`
+#' * \code{\link{download_huc}}: `"huc"`
+#' * \code{\link{download_cropscape}}: `"cropscape"`, `"cdl"`
+#' * \code{\link{download_prism}}: `"prism"`
 #' @return NULL
 #' @examples
 #' \dontrun{
@@ -57,7 +60,7 @@ download_data <-
                      "sedac_population", "groads", "population",
                      "hms", "smoke", "tri", "nei",
                      "gridmet", "terraclimate", "huc", "cropscape", "cdl",
-                     "prism", "olm", "openlandmap"),
+                     "prism"),
     directory_to_save = NULL,
     acknowledgement = FALSE,
     ...
@@ -94,9 +97,7 @@ download_data <-
       huc = download_huc,
       cropscape = download_cropscape,
       cdl = download_cropscape,
-      prism = download_prism,
-      olm = download_olm,
-      openlandmap = download_olm
+      prism = download_prism
     )
 
     tryCatch(
@@ -2819,184 +2820,6 @@ download_nei <- function(
 }
 
 
-# nolint start
-#' Download OpenLandMap data
-#' @description
-#' Accesses and downloads OpenLandMap data from the [OpenLandMap website](https://opengeohub.org/about-openlandmap/).
-#' @param product character(1). Available collection name in OpenLandMap
-#' STAC Catalog. [list_stac_files] with `id_only = TRUE` to see available collections.
-#' * "no2_s5p.l3.trop.tmwm"
-#' * "no2_s5p.l3.trop.tmwm.ltm"
-#' * "log.oc_iso.10694"
-#' * "evi_mod13q1.stl.trend.logit.ols.beta"
-#' * "land.cover_esacci.lc.l4"
-#' * "evi_mod13q1.tmwm.inpaint"
-#' * "dtm.bareearth_ensemble"
-#' * "fapar_essd.lstm"
-#' * "fapar_essd.lstm.p95.beta"
-#' * "pot.fapar_fapar.p95.eml.m"
-#' * "pot.fapar_fapar.p95.eml"
-#' * "snow.cover_esa.modis"
-#' * "snow.cover_esa.modis.ltm"
-#' * "wilderness_li2022.human.footprint"
-#' * "wv_mcd19a2v061.seasconv"
-#' * "wv_mcd19a2v061.seasconv.m_p50"
-#' * "wv_mcd19a2v061.seasconv.m_p25"
-#' * "wv_mcd19a2v061.seasconv.m_p75"
-#' * "wv_mcd19a2v061.seasconv.m_std"
-#' * "wv_mcd19a2v061.seasconv.m.yearly"
-#' * "bulkdens.fineearth_usda.4a1h"
-#' * "geom_merit.dem"
-#' * "fapar_proba.v"
-#' * "forest.cover_esacci.ifl"
-#' * "grtgroup_usda.soiltax"
-#' * "land.cover_copernicus"
-#' * "organic.carbon.stock_msa.kgm2"
-#' * "organic.carbon_usda.6a1c"
-#' * "pft.landcover_esa.cci.lc"
-#' * "precipitation_sm2rain.ltm"
-#' * "ph.h2o_usda.4c1a2a"
-#' * "pop.count_ghs.jrc"
-#' * "sand.wfraction_usda.3a1a1a"
-#' * "lc_mcd12q1v061.p1"
-#' * "texture.class_usda.tt"
-#' * "water.occurrence_jrc.surfacewater"
-#' * "watercontent.33kPa_usda.4b1c"
-#' * "dsm_glo30"
-#' * "lc_glad.glcluc"
-#' * "lc_glad.glcluc.change"
-#' * "landuse.cropland_hyde"
-#' * "landuse.pasture_hyde"
-#' * "land.use.land.cover_hilda.plus"
-#' * "lst_mod11a2.daytime.trend.logit.ols.beta"
-#' * "lst_mod11a2.nighttime.trend.logit.ols.beta"
-#' * "lst_mod11a2.daytime.annual"
-#' * "lst_mod11a2.nighttime.annual"
-#' * "lst_mod11a2.daytime"
-#' * "lst_mod11a2.nighttime"
-#' * "landform_usgs.ecotapestry"
-#' * "lithology_usgs.ecotapestry"
-#' * "grtgroup_usda.soiltax.hapludalfs"
-#' * "biome.type_biome00k"
-#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest"
-#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp26"
-#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp45"
-#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp85"
-#' * "biomes_biome6k.tropical.savanna"
-#' * "biomes_biome6k.tropical.savanna.rcp26"
-#' * "biomes_biome6k.tropical.savanna.rcp45"
-#' * "biomes_biome6k.tropical.savanna.rcp85"
-#' * "lc_glc.fcs30d"
-#' * "nightlights.average_viirs.v21"
-#' * "nightlights.difference_viirs.v21"
-#' * "l2a.gedi"
-#' * "fluxnet"
-#' * "gbov"
-#' * "geowiki.lc"
-#' * "geowiki.forest.loss"
-#' * "veg.plot"
-#' * "obis"
-#' * "fapar.eml"
-#' @param format character(1). File format to query. Default is "tif".
-#' It could be used as a pattern search for the file names.
-#' @param directory_to_save character(1). Directory to download files.
-#' @param acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledges that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Insang Song
-#' @note `extdata/openlandmap_assets.rds` contains the available assets in OpenLandMap.
-#' Users may want to check the available assets to download data directly.
-#' For developers: JSON files should be found at STAC catalog of OpenLandMap when updated.
-#' @return NULL; GeoTIFF (.tif) files will be stored in
-#' \code{directory_to_save}.
-#' @seealso [list_stac_files]
-#' @importFrom Rdpack reprompt
-#' @references
-#' \insertRef{data_hengl2023openlandmap}{amadeus}
-#' @examples
-#' \dontrun{
-#' download_olm(
-#'   product = "no2_s5p.l3.trop.tmwm",
-#'   format = "tif",
-#'   directory_to_save = "./data",
-#'   acknowledgement = TRUE,
-#'   download = TRUE,
-#'   remove_command = TRUE
-#' )
-#' }
-#' @export
-# nolint end
-download_olm <- function(
-  product = NULL,
-  format = "tif",
-  directory_to_save = NULL,
-  acknowledgement = FALSE,
-  download = FALSE,
-  remove_command = FALSE
-) {
-  #### 1. check for data download acknowledgement
-  download_permit(acknowledgement = acknowledgement)
-  #### 2. directory setup
-  download_setup_dir(directory_to_save)
-  directory_to_save <- download_sanitize_path(directory_to_save)
-
-  #### 3. define measurement data paths
-
-  download_urls <-
-    list_stac_files(
-      which = product,
-      format = format,
-      id_only = FALSE
-    )
-  url_filenames <- strsplit(download_urls, "/", fixed = TRUE)[[1]]
-  url_filenames <- sapply(url_filenames, \(x) x[length(x)])
-
-  download_names <- paste0(directory_to_save, url_filenames)
-  #### 4. build download command
-  download_commands <-
-    paste0("wget -e robots=off -np ",
-           download_urls,
-           " -O ",
-           download_names,
-           "\n")
-
-  #### 5. initiate "..._curl_commands.txt"
-  commands_txt <- paste0(
-    directory_to_save,
-    "OLM_queried_",
-    product,
-    "_",
-    Sys.Date(),
-    "_wget_commands.txt"
-  )
-  download_sink(commands_txt)
-  #### 6. concatenate and print download commands to "..._curl_commands.txt"
-  writeLines(download_commands)
-  #### 7. finish "..._curl_commands.txt" file
-  sink()
-  #### 8. build system command
-  system_command <- paste0(
-    ". ",
-    commands_txt,
-    "\n"
-  )
-  #### 9. download data
-  download_run(download = download,
-               system_command = system_command)
-
-  message("Requests were processed.\n")
-  #### 10. remove download commands
-  download_remove_command(commands_txt = commands_txt,
-                          remove = remove_command)
-
-}
-
 
 # nolint start
 #' Download National Hydrography Dataset (NHD) data

diff --git a/R/download_auxiliary.R b/R/download_auxiliary.R
@@ -451,71 +451,6 @@ test_download_functions <- function(
   }
 }
 
-#' Retrieve file links from SpatioTemporal Assets Catalog (STAC)
-#' @description
-#' Read file links from SpatioTemporal Assets Catalog (STAC) JSON file.
-#' @note
-#' Retrieving URLs may take a while depending on the spatial tiling,
-#' temporal resolution, and other assets. Users are encouraged to use
-#' `which` parameter to select a specific collection.
-#' @param stac_json character(1). Full path of STAC JSON file.
-#' @param format character(1). Format of target files. Default is "tif".
-#' @param which numeric/character. Index or name of collection to retrieve.
-#' @param id_only logical(1). Return collection IDs only.
-#' @return character vector of file links.
-#' @examples
-#' \dontrun{
-#' read_stac_json()
-#' }
-#' @export
-#' @keywords auxiliary
-#' @author Insang Song
-#' @importFrom rstac read_collections
-#' @importFrom rstac read_items
-#' @importFrom rstac assets_url
-#' @importFrom rstac read_stac
-#' @importFrom rstac links
-list_stac_files <-
-  function(
-    stac_json =
-    "https://s3.eu-central-1.wasabisys.com/stac/openlandmap/catalog.json",
-    format = "tif",
-    which = NULL,
-    id_only = FALSE
-  ) {
-    allcollections <-
-      rstac::read_stac(
-        stac_json
-      ) |>
-      rstac:::read_collections.catalog()
-    collection <- allcollections$collections
-    collection_ids <- sapply(collection, \(x) x$id)
-    if (id_only) {
-      return(collection_ids)
-    }
-
-    message(
-      sprintf(
-        "Names of collections include: %s",
-        paste(
-          collection_ids, collapse = "\n"
-        )
-      )
-    )
-
-    which <- if (!is.numeric(which)) grep(which, collection_ids) else which
-    collection <- if (is.null(which)) collection else collection[which]
-
-    collection_assets <-
-      lapply(
-        collection,
-        \(x) rstac::assets_url(rstac:::read_items.doc_collection(x))
-      )
-    collection_assets <- unlist(collection_assets)
-    list_assets <- grep(sprintf("%s$", format), collection_assets, value = TRUE)
-
-    return(list_assets)
-  }
 
 #' Sort NOAA NARR variables
 #' @description