[DO NOT MERGE] Re-worked tooling for Python environments in REMIND

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '1053587'
+ValidationKey: '1196520'
 AutocreateReadme: yes
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
  - id: mixed-line-ending
 
 - repo: https://github.com/lorenzwalthert/precommit
- rev: 7910e0323d7213f34275a7a562b9ef0fde8ce1b9 # frozen: v0.4.2
+ rev: bae853d82da476eee0e0a57960ee6b741a3b3fb7 # frozen: v0.4.3
  hooks:
  - id: parsable-R
  - id: deps-in-desc

CITATION.cff

@@ -2,14 +2,17 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'piamenv: Package environment support for PIAM'
-version: 0.5.3
-date-released: '2024-06-05'
+version: 0.6.0
+date-released: '2024-08-07'
 abstract: Enables easier management of package environments, based on renv and Python
  venv.
 authors:
 - family-names: Sauer
  given-names: Pascal
  email: [email protected]
+- family-names: Rüter
+ given-names: Tonn
+ email: [email protected]
 license: LGPL-3.0
 repository-code: https://github.com/pik-piam/piamenv
 

DESCRIPTION

@@ -1,20 +1,25 @@
 Type: Package
 Package: piamenv
 Title: Package environment support for PIAM
-Version: 0.5.3
-Date: 2024-06-05
-Authors@R:
- person("Pascal", "Sauer", , "[email protected]", role = c("aut", "cre"))
+Version: 0.6.0
+Date: 2024-08-07
+Authors@R: c(
+ person("Pascal", "Sauer", , "[email protected]", role = c("aut", "cre")),
+ person("Tonn", "Rüter", , "[email protected]", role = c("aut")))
 Description: Enables easier management of package environments, based on renv and Python venv.
 License: LGPL-3
 URL: https://github.com/pik-piam/piamenv
 Imports:
  desc,
  methods,
  renv,
- withr
+ reticulate,
+ stringr,
+ purrr,
+ withr,
+ yaml
 Suggests:
  covr,
  testthat
 Encoding: UTF-8
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.2

NAMESPACE

@@ -1,14 +1,30 @@
 # Generated by roxygen2: do not edit by hand
 
+export(archivePythonEnv)
 export(archiveRenv)
 export(checkDeps)
-export(createResultsfolderPythonVirtualEnv)
+export(checkPythonDeps)
+export(checkPythonEnv)
+export(checkPythonRequirements)
 export(fixDeps)
-export(pythonBinPath)
 export(restoreRenv)
 export(revertDevelopmentVersions)
 export(showUpdates)
 export(stopIfLoaded)
-export(updatePythonVirtualEnv)
 export(updateRenv)
-export(writePythonVirtualEnvLockFile)
+importFrom(purrr,map)
+importFrom(purrr,map_chr)
+importFrom(purrr,pmap)
+importFrom(reticulate,import)
+importFrom(reticulate,py_config)
+importFrom(reticulate,py_discover_config)
+importFrom(reticulate,py_list_packages)
+importFrom(reticulate,use_condaenv)
+importFrom(reticulate,use_python)
+importFrom(reticulate,use_virtualenv)
+importFrom(stringr,regex)
+importFrom(stringr,str_match)
+importFrom(utils,head)
+importFrom(utils,strcapture)
+importFrom(yaml,as.yaml)
+importFrom(yaml,yaml.load)

R/archivePythonEnv.R

@@ -0,0 +1,64 @@
+#' writePythonVirtualEnvLockFile
+#'
+#' Write Python virtual environment lock file specifying package versions currently installed in the given
+#' virtual environment. Using the lock file, a new virtual environment can be built containing exactly the same
+#' package versions.
+#'
+#' @param outputDir Path to the directory where the lock file will be written
+#' @param pythonConfig as produced by \code{\link{checkPythonEnv}}. Defaults to NULL. If not provided, the
+#' RETICULATE_PYTHON environment variable is used
+#'
+#' @author Mika Pflüger, Tonn Rüter
+#' @importFrom reticulate py_config
+#' @importFrom yaml as.yaml yaml.load
+#' @importFrom utils head
+#' @export
+archivePythonEnv <- function(outputDir, pythonConfig = reticulate::py_config()) {
+ if (!dir.exists(outputDir)) {
+ stop("Provided path '", outputDir, "' does not exist.")
+ }
+ dateAndTime <- format(Sys.time(), "%y%m%d_%H%M%S")
+ # Get the path to the Python executable
+ # Check if (ana)conda, venv or system Python is used
+ if (any(as.logical(c(pythonConfig$conda, pythonConfig$anaconda)))) {
+ envType <- "conda"
+ logFile <- file.path(outputDir, paste0("log_pyenv_", dateAndTime, "_", envType, ".txt"))
+ # First run the standard conda command to create an environment file it is yaml formatted, which is nice 'cause
+ # we possibly need to manipulate it later ..
+ condaEnv <- yaml.load(system2("conda", args = c("env", "export"), stdout = TRUE))
+ # .. in case some packages are installed via pip, we need to replace them with the actual pip freeze output
+ # because the conda env export command does not include information about packages that have been installed
+ # from particular sources repositories (i.e. it would show a version string like "0.1.4a0" instead of the
+ # " @ git+https://<repo-url>/<package name>.git@<commit-hash>"). However, this is important for reproducibility and
+ # a open issue with conda: https://github.com/conda/conda/issues/10320
+ # Let's start by checking whether there are any pip dependencies in the first place. Conda file contain those in
+ # a dedicated section called "pip". If there are none, we can skip this step.
+ pipIdx <- head(which(vapply(condaEnv$dependencies, function(x) "pip" %in% names(x), FUN.VALUE = logical(1))), 1)
+ # This vector should not contain more than one element. If there'd be more than one element, there's something
+ # wrong with the conda environment file. Don't fail explicitly here, just disregard other pip dependencies except
+ # the first one.
+ if (!is.null(pipIdx)) {
+ # Run python -m pip [...] here instead of pip [...] cause from within the conda environment, there a chance
+ # that system pip is used instead of pip that comes with the conda installation
+ pipDeps <- system2("python", args = c("-m pip", "freeze", "--local"), stdout = TRUE)
+ condaEnv$dependencies[[pipIdx]]$pip <- pipDeps
+ }
+ # Dump conda environment + improved pip dependencies to yaml string
+ envAsString <- as.yaml(condaEnv)
+ } else {
+ if (pythonConfig$virtualenv != "") {
+ envType <- "venv"
+ # Need to use --local to avoid listing globally installed packages
+ archiveArgs <- c("-m pip", "freeze", "--local")
+ } else {
+ envType <- "system"
+ archiveArgs <- c("-m pip", "freeze")
+ }
+ envAsString <- system2(normalizePath(pythonConfig$python), args = archiveArgs, stdout = TRUE)
+ }
+ logFile <- normalizePath(file.path(outputDir, paste0("log_pyenv_", dateAndTime, "_", envType, ".txt")),
+ mustWork = FALSE
+ )
+ writeLines(envAsString, logFile)
+ return(invisible(TRUE))
+}