From 27233616aa9996b5ac63eb5fd14612ffd36f5434 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Mon, 2 Oct 2023 10:19:08 -0400 Subject: [PATCH] Update documentation (#90) * Updating documentation * Add Linux keyring guidance to README * Revise vignette eval flags --- DESCRIPTION | 6 +- R/Execution.R | 68 +- R/ModuleEnv.R | 4 +- R/ModuleInstantiation.R | 4 +- R/ResultModelCreation.R | 73 +- R/ResultsUpload.R | 154 ++- R/RunModule.R | 115 +- README.md | 16 +- docs/404.html | 4 +- .../CreatingAnalysisSpecification.html | 245 +++- docs/articles/CreatingModules.html | 88 +- docs/articles/ExecuteStrategus.html | 6 +- docs/articles/IntroductionToStrategus.html | 22 +- docs/articles/index.html | 4 +- docs/authors.html | 16 +- docs/index.html | 13 +- docs/news/index.html | 4 +- docs/pkgdown.yml | 4 +- docs/pull_request_template.html | 4 +- docs/reference/Strategus-package.html | 5 +- docs/reference/addModuleSpecifications.html | 4 +- docs/reference/addSharedResources.html | 4 +- .../reference/createCdmExecutionSettings.html | 32 +- .../createEmptyAnalysisSpecificiations.html | 4 +- docs/reference/createResultDataModels.html | 147 ++ .../createResultsExecutionSettings.html | 16 +- docs/reference/dot-nullList.html | 105 ++ .../ensureAllModulesInstantiated.html | 4 +- docs/reference/execute.html | 4 +- docs/reference/getModuleList.html | 4 +- docs/reference/index.html | 17 +- docs/reference/retrieveConnectionDetails.html | 4 +- docs/reference/runSchemaCreation.html | 145 ++ docs/reference/storeConnectionDetails.html | 4 +- docs/reference/unlockKeyring.html | 4 +- docs/reference/withModuleRenv.html | 161 +++ docs/sitemap.xml | 12 + extras/PackageMaintenance.R | 1 + extras/Strategus.pdf | Bin 122937 -> 140210 bytes inst/csv/modules.csv | 2 +- inst/doc/CreatingAnalysisSpecification.pdf | Bin 0 -> 260712 bytes inst/doc/CreatingModules.pdf | Bin 0 -> 220983 bytes inst/doc/ExecuteStrategus.pdf | Bin 0 -> 228442 bytes inst/doc/IntroductionToStrategus.pdf | Bin 0 -> 171525 bytes man/Strategus-package.Rd | 4 +- tests/testthat/setup.R | 42 +- vignettes/CreatingAnalysisSpecification.Rmd | 22 +- vignettes/CreatingModules.Rmd | 34 +- vignettes/IntroductionToStrategus.Rmd | 2 +- .../extras/CreateJobContextForTesting.R | 143 ++ vignettes/module_skeleton/renv/.gitignore | 7 + vignettes/module_skeleton/renv/activate.R | 1201 +++++++++++++++++ vignettes/module_skeleton/renv/settings.dcf | 10 + vignettes/module_skeleton/renv/settings.json | 19 + .../module_skeleton/tests/test-eunomia.R | 41 + .../module_skeleton/tests/testJobContext.rds | Bin 0 -> 2015 bytes vignettes/module_skeleton/tests/testScript.R | 10 + 57 files changed, 2668 insertions(+), 396 deletions(-) create mode 100644 docs/reference/createResultDataModels.html create mode 100644 docs/reference/dot-nullList.html create mode 100644 docs/reference/runSchemaCreation.html create mode 100644 docs/reference/withModuleRenv.html create mode 100644 inst/doc/CreatingAnalysisSpecification.pdf create mode 100644 inst/doc/CreatingModules.pdf create mode 100644 inst/doc/ExecuteStrategus.pdf create mode 100644 inst/doc/IntroductionToStrategus.pdf create mode 100644 vignettes/module_skeleton/extras/CreateJobContextForTesting.R create mode 100644 vignettes/module_skeleton/renv/.gitignore create mode 100644 vignettes/module_skeleton/renv/activate.R create mode 100644 vignettes/module_skeleton/renv/settings.dcf create mode 100644 vignettes/module_skeleton/renv/settings.json create mode 100644 vignettes/module_skeleton/tests/test-eunomia.R create mode 100644 vignettes/module_skeleton/tests/testJobContext.rds create mode 100644 vignettes/module_skeleton/tests/testScript.R diff --git a/DESCRIPTION b/DESCRIPTION index f23ebde2..a19bfc14 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,10 +2,10 @@ Package: Strategus Type: Package Title: Coordinating and Executing Analytics Using HADES Modules Version: 0.1.0 -Date: 2023-08-15 +Date: 2023-09-25 Authors@R: c( - person("Martijn", "Schuemie", email = "schuemie@ohdsi.org", role = c("aut", "cre")), - person("Anthony", "Sena", email = "sena@ohdsi.org", role = c("aut")), + person("Martijn", "Schuemie", email = "schuemie@ohdsi.org", role = c("aut")), + person("Anthony", "Sena", email = "sena@ohdsi.org", role = c("aut", "cre")), person("Jamie", "Gilbert", role = c("aut")), person("Observational Health Data Science and Informatics", role = c("cph")) ) diff --git a/R/Execution.R b/R/Execution.R index 44c0ca5a..c4e29680 100644 --- a/R/Execution.R +++ b/R/Execution.R @@ -102,32 +102,33 @@ generateTargetsScript <- function(analysisSpecifications, executionSettings, dep } ### Note anything inisde this block will be scoped inside the targets script file - targets::tar_script({ - ## - # Generated by Strategus - not advisable to edit by hand - ## - analysisSpecificationsLoad <- readRDS(analysisSpecificationsFileName) - moduleToTargetNames <- readRDS(moduleToTargetNamesFileName) - dependencies <- readRDS(dependenciesFileName) - - targets::tar_option_set(packages = c("Strategus", "keyring"), imports = c("Strategus", "keyring")) - targetList <- list( - targets::tar_target(analysisSpecifications, readRDS(analysisSpecificationsFileName)), - # NOTE Execution settings could be mapped to many different cdms making re-execution across cdms much simpler - targets::tar_target(executionSettings, readRDS(executionSettingsFileName)), - targets::tar_target(keyringSettings, readRDS(keyringSettingsFileName)) - ) - - # factory for producing module targets based on their dependencies - # This could be inside Strategus as an exported function - # it would also be much cleaner to use a targets pattern = cross(analysisSpecifications$moduleSpecifications) - # however, working out how to handle dependencies wasn't obvious - # This approach could be modified to allow multiple executionSettings, but that would require a substantial re-write - for (i in 1:length(analysisSpecificationsLoad$moduleSpecifications)) { - moduleSpecification <- analysisSpecificationsLoad$moduleSpecifications[[i]] - targetName <- sprintf("%s_%d", moduleSpecification$module, i) - dependencyModules <- dependencies[dependencies$module == moduleSpecification$module,]$dependsOn - dependencyTargetNames <- moduleToTargetNames[moduleToTargetNames$module %in% dependencyModules,]$targetName + targets::tar_script( + { + ## + # Generated by Strategus - not advisable to edit by hand + ## + analysisSpecificationsLoad <- readRDS(analysisSpecificationsFileName) + moduleToTargetNames <- readRDS(moduleToTargetNamesFileName) + dependencies <- readRDS(dependenciesFileName) + + targets::tar_option_set(packages = c("Strategus", "keyring"), imports = c("Strategus", "keyring")) + targetList <- list( + targets::tar_target(analysisSpecifications, readRDS(analysisSpecificationsFileName)), + # NOTE Execution settings could be mapped to many different cdms making re-execution across cdms much simpler + targets::tar_target(executionSettings, readRDS(executionSettingsFileName)), + targets::tar_target(keyringSettings, readRDS(keyringSettingsFileName)) + ) + + # factory for producing module targets based on their dependencies + # This could be inside Strategus as an exported function + # it would also be much cleaner to use a targets pattern = cross(analysisSpecifications$moduleSpecifications) + # however, working out how to handle dependencies wasn't obvious + # This approach could be modified to allow multiple executionSettings, but that would require a substantial re-write + for (i in 1:length(analysisSpecificationsLoad$moduleSpecifications)) { + moduleSpecification <- analysisSpecificationsLoad$moduleSpecifications[[i]] + targetName <- sprintf("%s_%d", moduleSpecification$module, i) + dependencyModules <- dependencies[dependencies$module == moduleSpecification$module, ]$dependsOn + dependencyTargetNames <- moduleToTargetNames[moduleToTargetNames$module %in% dependencyModules, ]$targetName # Use of tar_target_raw allows dynamic names targetList[[length(targetList) + 1]] <- targets::tar_target_raw(targetName, @@ -148,9 +149,11 @@ generateTargetsScript <- function(analysisSpecifications, executionSettings, dep } } targetList - }, script = fileName) + }, + script = fileName + ) - #Store settings objects in the temp folder so they are available in targets + # Store settings objects in the temp folder so they are available in targets analysisSpecificationsFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "analysisSpecifications.rds")) saveRDS(analysisSpecifications, analysisSpecificationsFileName) executionSettingsFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "executionSettings.rds")) @@ -175,9 +178,11 @@ generateTargetsScript <- function(analysisSpecifications, executionSettings, dep dependenciesFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "dependencies.rds")) saveRDS(dependencies, dependenciesFileName) - execResultsUpload <- all(c(is(executionSettings, "CdmExecutionSettings"), - !is.null(executionSettings$resultsConnectionDetailsReference), - !is.null(executionSettings$resultsDatabaseSchema))) + execResultsUpload <- all(c( + is(executionSettings, "CdmExecutionSettings"), + !is.null(executionSettings$resultsConnectionDetailsReference), + !is.null(executionSettings$resultsDatabaseSchema) + )) # Settings required inside script. There is probably a much cleaner way of doing this writeLines(c( @@ -192,4 +197,3 @@ generateTargetsScript <- function(analysisSpecifications, executionSettings, dep return(fileName) } - diff --git a/R/ModuleEnv.R b/R/ModuleEnv.R index 995eb823..b26de8ba 100644 --- a/R/ModuleEnv.R +++ b/R/ModuleEnv.R @@ -14,12 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -.handleInjectVar <- function (x) { +.handleInjectVar <- function(x) { hVar <- function(x) { if (is.character(x)) { return(sprintf('"%s"', x)) } else { - return(sprintf('%s', x)) + return(sprintf("%s", x)) } } diff --git a/R/ModuleInstantiation.R b/R/ModuleInstantiation.R index 9cca1720..df27152f 100644 --- a/R/ModuleInstantiation.R +++ b/R/ModuleInstantiation.R @@ -194,10 +194,10 @@ instantiateModule <- function(module, version, remoteRepo, remoteUsername, modul withModuleRenv( code = { ParallelLogger::addDefaultFileLogger( - fileName = file.path(moduleFolder, 'moduleInitLog.txt') + fileName = file.path(moduleFolder, "moduleInitLog.txt") ) ParallelLogger::addDefaultErrorReportLogger( - fileName = file.path(moduleFolder, 'moduleInitErrorReport.txt') + fileName = file.path(moduleFolder, "moduleInitErrorReport.txt") ) renv::restore(prompt = FALSE) }, diff --git a/R/ResultModelCreation.R b/R/ResultModelCreation.R index 78f988d4..2446561a 100644 --- a/R/ResultModelCreation.R +++ b/R/ResultModelCreation.R @@ -53,22 +53,23 @@ createResultDataModels <- function(analysisSpecifications, ## # Code execution inside targets block ## - targets::tar_script({ - ## - # Generated by Strategus - not advisable to edit by hand - ## - analysisSpecificationsLoad <- readRDS(analysisSpecificationsFileName) - - targets::tar_option_set(packages = c("Strategus", "keyring"), imports = c("Strategus", "keyring")) - targetList <- list( - targets::tar_target(analysisSpecifications, readRDS(analysisSpecificationsFileName)), - targets::tar_target(executionSettings, readRDS(executionSettingsFileName)), - targets::tar_target(keyringSettings, readRDS(keyringSettingsFileName)) - ) - - for (i in 1:length(analysisSpecificationsLoad$moduleSpecifications)) { - moduleSpecification <- analysisSpecificationsLoad$moduleSpecifications[[i]] - targetName <- sprintf("%s_%d_schema_creation", moduleSpecification$module, i) + targets::tar_script( + { + ## + # Generated by Strategus - not advisable to edit by hand + ## + analysisSpecificationsLoad <- readRDS(analysisSpecificationsFileName) + + targets::tar_option_set(packages = c("Strategus", "keyring"), imports = c("Strategus", "keyring")) + targetList <- list( + targets::tar_target(analysisSpecifications, readRDS(analysisSpecificationsFileName)), + targets::tar_target(executionSettings, readRDS(executionSettingsFileName)), + targets::tar_target(keyringSettings, readRDS(keyringSettingsFileName)) + ) + + for (i in 1:length(analysisSpecificationsLoad$moduleSpecifications)) { + moduleSpecification <- analysisSpecificationsLoad$moduleSpecifications[[i]] + targetName <- sprintf("%s_%d_schema_creation", moduleSpecification$module, i) # Use of tar_target_raw allows dynamic names targetList[[length(targetList) + 1]] <- targets::tar_target_raw(targetName, @@ -83,7 +84,7 @@ createResultDataModels <- function(analysisSpecifications, script = script ) - #Store settings objects in the temp folder so they are available in targets + # Store settings objects in the temp folder so they are available in targets analysisSpecificationsFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "analysisSpecifications.rds")) saveRDS(analysisSpecifications, analysisSpecificationsFileName) executionSettingsFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "executionSettings.rds")) @@ -175,29 +176,28 @@ runSchemaCreation <- function(analysisSpecifications, keyringSettings, moduleInd createDataModelSchema <- NULL getDataModelSpecifications <- function(...) { - if (file.exists('resultsDataModelSpecification.csv')) { + if (file.exists("resultsDataModelSpecification.csv")) { res <- CohortGenerator::readCsv( - file = 'resultsDataModelSpecification.csv' + file = "resultsDataModelSpecification.csv" ) return(res) } return(NULL) } - source('Main.R') + source("Main.R") jobContext <- readRDS(jobContextFileName) specifications <- getDataModelSpecifications(jobContext) - if (Sys.getenv('FORCE_RENV_USE', '') == 'TRUE') { - renv::use(lockfile = 'renv.lock') + if (Sys.getenv("FORCE_RENV_USE", "") == "TRUE") { + renv::use(lockfile = "renv.lock") } - ParallelLogger::addDefaultFileLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, 'log.txt')) - ParallelLogger::addDefaultErrorReportLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, 'errorReportR.txt')) + ParallelLogger::addDefaultFileLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, "log.txt")) + ParallelLogger::addDefaultErrorReportLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, "errorReportR.txt")) # Main.R can override default behaviour by implementing this function if (is.function(createDataModelSchema)) { - # If the keyring is locked, unlock it, set the value and then re-lock it keyringName <- jobContext$keyringSettings$keyringName keyringLocked <- Strategus::unlockKeyring(keyringName = keyringName) @@ -212,7 +212,7 @@ runSchemaCreation <- function(analysisSpecifications, keyringSettings, moduleInd if (keyringLocked) { keyring::keyring_lock(keyring = keyringName) } - writeLines('schema.created', doneFile) + writeLines("schema.created", doneFile) } else if (is.data.frame(specifications)) { # Export schema to readable location CohortGenerator::writeCsv( @@ -220,7 +220,7 @@ runSchemaCreation <- function(analysisSpecifications, keyringSettings, moduleInd file = dataModelExportPath, warnOnCaseMismatch = FALSE ) - writeLines('specifications.written', doneFile) + writeLines("specifications.written", doneFile) } else { warning("Module does not include data specifications file or createDataModelSchema function") CohortGenerator::writeCsv( @@ -228,11 +228,11 @@ runSchemaCreation <- function(analysisSpecifications, keyringSettings, moduleInd file = dataModelExportPath, warnOnCaseMismatch = FALSE ) - writeLines('specifications.not.written', doneFile) + writeLines("specifications.not.written", doneFile) } - ParallelLogger::unregisterLogger('DEFAULT_FILE_LOGGER', silent = TRUE) - ParallelLogger::unregisterLogger('DEFAULT_ERRORREPORT_LOGGER', silent = TRUE) + ParallelLogger::unregisterLogger("DEFAULT_FILE_LOGGER", silent = TRUE) + ParallelLogger::unregisterLogger("DEFAULT_ERRORREPORT_LOGGER", silent = TRUE) }, moduleFolder = moduleFolder, tempScriptFile = file.path(moduleExecutionSettings$workSubFolder, "SchemaCreation.R"), @@ -244,8 +244,8 @@ runSchemaCreation <- function(analysisSpecifications, keyringSettings, moduleInd ) workStatus <- readLines(doneFile) - if (workStatus == 'specifications.written') { - moduleInfo <- ParallelLogger::loadSettingsFromJson(file.path(moduleFolder, 'MetaData.json')) + if (workStatus == "specifications.written") { + moduleInfo <- ParallelLogger::loadSettingsFromJson(file.path(moduleFolder, "MetaData.json")) keyringName <- jobContext$keyringSettings$keyringName keyringLocked <- Strategus::unlockKeyring(keyringName = keyringName) @@ -258,14 +258,15 @@ runSchemaCreation <- function(analysisSpecifications, keyringSettings, moduleInd sql <- ResultModelManager::generateSqlSchema(csvFilepath = dataModelExportPath) DatabaseConnector::renderTranslateExecuteSql(connection, - sql, - table_prefix = moduleInfo$TablePrefix, - database_schema = jobContext$moduleExecutionSettings$resultsDatabaseSchema) + sql, + table_prefix = moduleInfo$TablePrefix, + database_schema = jobContext$moduleExecutionSettings$resultsDatabaseSchema + ) if (keyringLocked) { keyring::keyring_lock(keyring = keyringName) } - } else if (workStatus == 'schema.created') { + } else if (workStatus == "schema.created") { message("Result schema creation handled inside module execution envrionment") } else { message("Schema not created for module") diff --git a/R/ResultsUpload.R b/R/ResultsUpload.R index 85abd85a..0a3ad60a 100644 --- a/R/ResultsUpload.R +++ b/R/ResultsUpload.R @@ -58,74 +58,76 @@ runResultsUpload <- function(analysisSpecifications, keyringSettings, moduleInde ## # Module space executed code ## - withModuleRenv({ - uploadResultsCallback <- NULL - - getDataModelSpecifications <- function(...) { - ParallelLogger::logInfo("Getting result model specification") - if (file.exists('resultsDataModelSpecification.csv')) { - res <- CohortGenerator::readCsv( - file = 'resultsDataModelSpecification.csv' - ) - return(res) + withModuleRenv( + { + uploadResultsCallback <- NULL + + getDataModelSpecifications <- function(...) { + ParallelLogger::logInfo("Getting result model specification") + if (file.exists("resultsDataModelSpecification.csv")) { + res <- CohortGenerator::readCsv( + file = "resultsDataModelSpecification.csv" + ) + return(res) + } + ParallelLogger::logInfo("No result model specification found") + return(NULL) } - ParallelLogger::logInfo("No result model specification found") - return(NULL) - } - source('Main.R') - moduleInfo <- ParallelLogger::loadSettingsFromJson('MetaData.json') - jobContext <- readRDS(jobContextFileName) - specifications <- getDataModelSpecifications(jobContext) - ParallelLogger::addDefaultFileLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, 'log.txt')) - ParallelLogger::addDefaultErrorReportLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, 'errorReportR.txt')) - - if (Sys.getenv('FORCE_RENV_USE', '') == 'TRUE') { - renv::use(lockfile = 'renv.lock') - } - - # Override default behaviour and do module specific upload inside module context? - if (is.function(uploadResultsCallback)) { - ParallelLogger::logInfo("Calling module result upload functionality") - # If the keyring is locked, unlock it, set the value and then re-lock it - ParallelLogger::logInfo("-- Getting result database credentials") - keyringName <- jobContext$keyringSettings$keyringName - keyringLocked <- Strategus::unlockKeyring(keyringName = keyringName) - resultsConnectionDetails <- keyring::key_get(jobContext$moduleExecutionSettings$resultsConnectionDetailsReference, keyring = keyringName) - resultsConnectionDetails <- ParallelLogger::convertJsonToSettings(resultsConnectionDetails) - resultsConnectionDetails <- do.call(DatabaseConnector::createConnectionDetails, resultsConnectionDetails) - jobContext$moduleExecutionSettings$resultsConnectionDetails <- resultsConnectionDetails - ParallelLogger::logInfo("-- Executing upload callback") - uploadResultsCallback(jobContext) - if (keyringLocked) { - keyring::keyring_lock(keyring = keyringName) + source("Main.R") + moduleInfo <- ParallelLogger::loadSettingsFromJson("MetaData.json") + jobContext <- readRDS(jobContextFileName) + specifications <- getDataModelSpecifications(jobContext) + ParallelLogger::addDefaultFileLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, "log.txt")) + ParallelLogger::addDefaultErrorReportLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, "errorReportR.txt")) + + if (Sys.getenv("FORCE_RENV_USE", "") == "TRUE") { + renv::use(lockfile = "renv.lock") } - ParallelLogger::logInfo("-- Upload completed") - writeLines('results.uploaded', doneFile) - } else if (is.null(specifications)) { - ParallelLogger::logInfo("No result specifications found, assuming module has produced no results") - # NO spect file Status - warning('data model specifications not loaded from module - skipping results upload') - writeLines('no.spec.found', doneFile) - } else { - # Spec file written - ParallelLogger::logInfo("Writing spec for result upload outside of module context") - CohortGenerator::writeCsv( - x = specifications, - file = dataModelExportPath, - warnOnFileNameCaseMismatch = FALSE - ) - writeLines('specifications.written', doneFile) - } - ParallelLogger::unregisterLogger('DEFAULT_FILE_LOGGER', silent = TRUE) - ParallelLogger::unregisterLogger('DEFAULT_ERRORREPORT_LOGGER', silent = TRUE) + # Override default behaviour and do module specific upload inside module context? + if (is.function(uploadResultsCallback)) { + ParallelLogger::logInfo("Calling module result upload functionality") + # If the keyring is locked, unlock it, set the value and then re-lock it + ParallelLogger::logInfo("-- Getting result database credentials") + keyringName <- jobContext$keyringSettings$keyringName + keyringLocked <- Strategus::unlockKeyring(keyringName = keyringName) + resultsConnectionDetails <- keyring::key_get(jobContext$moduleExecutionSettings$resultsConnectionDetailsReference, keyring = keyringName) + resultsConnectionDetails <- ParallelLogger::convertJsonToSettings(resultsConnectionDetails) + resultsConnectionDetails <- do.call(DatabaseConnector::createConnectionDetails, resultsConnectionDetails) + jobContext$moduleExecutionSettings$resultsConnectionDetails <- resultsConnectionDetails + ParallelLogger::logInfo("-- Executing upload callback") + uploadResultsCallback(jobContext) + if (keyringLocked) { + keyring::keyring_lock(keyring = keyringName) + } + ParallelLogger::logInfo("-- Upload completed") + writeLines("results.uploaded", doneFile) + } else if (is.null(specifications)) { + ParallelLogger::logInfo("No result specifications found, assuming module has produced no results") + # NO spect file Status + warning("data model specifications not loaded from module - skipping results upload") + writeLines("no.spec.found", doneFile) + } else { + # Spec file written + ParallelLogger::logInfo("Writing spec for result upload outside of module context") + CohortGenerator::writeCsv( + x = specifications, + file = dataModelExportPath, + warnOnFileNameCaseMismatch = FALSE + ) + writeLines("specifications.written", doneFile) + } - }, + ParallelLogger::unregisterLogger("DEFAULT_FILE_LOGGER", silent = TRUE) + ParallelLogger::unregisterLogger("DEFAULT_ERRORREPORT_LOGGER", silent = TRUE) + }, moduleFolder = moduleFolder, tempScriptFile = tempScriptFile, - injectVars = list(jobContextFileName = jobContextFileName, - dataModelExportPath = dataModelExportPath, - doneFile = doneFile) + injectVars = list( + jobContextFileName = jobContextFileName, + dataModelExportPath = dataModelExportPath, + doneFile = doneFile + ) ) ## # end Module executed code @@ -142,10 +144,10 @@ runResultsUpload <- function(analysisSpecifications, keyringSettings, moduleInde workStatus <- readLines(doneFile) - if (workStatus == 'specifications.written') { + if (workStatus == "specifications.written") { ParallelLogger::logInfo("Uploading results according to module specification") specifications <- CohortGenerator::readCsv(dataModelExportPath) - moduleInfo <- ParallelLogger::loadSettingsFromJson(file.path(moduleFolder, 'MetaData.json')) + moduleInfo <- ParallelLogger::loadSettingsFromJson(file.path(moduleFolder, "MetaData.json")) keyringName <- jobContext$keyringSettings$keyringName keyringLocked <- Strategus::unlockKeyring(keyringName = keyringName) @@ -157,22 +159,24 @@ runResultsUpload <- function(analysisSpecifications, keyringSettings, moduleInde jobContext$moduleExecutionSettings$resultsConnectionDetails <- resultsConnectionDetails ParallelLogger::logInfo("Calling RMM for upload") - ResultModelManager::uploadResults(connectionDetails = jobContext$moduleExecutionSettings$resultsConnectionDetails, - schema = jobContext$moduleExecutionSettings$resultsDatabaseSchema, - resultsFolder = jobContext$moduleExecutionSettings$resultsSubFolder, - tablePrefix = moduleInfo$TablePrefix, - forceOverWriteOfSpecifications = FALSE, - purgeSiteDataBeforeUploading = FALSE, - databaseIdentifierFile = 'database_meta_data.csv', - runCheckAndFixCommands = FALSE, - warnOnMissingTable = TRUE, - specifications = specifications) + ResultModelManager::uploadResults( + connectionDetails = jobContext$moduleExecutionSettings$resultsConnectionDetails, + schema = jobContext$moduleExecutionSettings$resultsDatabaseSchema, + resultsFolder = jobContext$moduleExecutionSettings$resultsSubFolder, + tablePrefix = moduleInfo$TablePrefix, + forceOverWriteOfSpecifications = FALSE, + purgeSiteDataBeforeUploading = FALSE, + databaseIdentifierFile = "database_meta_data.csv", + runCheckAndFixCommands = FALSE, + warnOnMissingTable = TRUE, + specifications = specifications + ) ParallelLogger::logInfo("Upload completed") if (keyringLocked) { keyring::keyring_lock(keyring = keyringName) } - } else if (workStatus == 'results.uploaded') { + } else if (workStatus == "results.uploaded") { message("Result upload handled inside module execution envrionment") } else { message("Results not uploaded for module") diff --git a/R/RunModule.R b/R/RunModule.R index 72c02f59..d7087afb 100644 --- a/R/RunModule.R +++ b/R/RunModule.R @@ -64,67 +64,68 @@ runModule <- function(analysisSpecifications, keyringSettings, moduleIndex, exec stop("Unhandled executionSettings class! Must be one of the following: CdmExecutionSettings, ResultsExecutionSettings") } withModuleRenv( - code = { - ############################ - # Generated by strategus - do not modify by hand - ############################ - source('Main.R') - jobContext <- readRDS(jobContextFileName) - - unlockKeyring <- function(keyringName) { - # If the keyring is locked, unlock it, set the value and then re-lock it - keyringLocked <- keyring::keyring_is_locked(keyring = keyringName) - if (keyringLocked) { - keyring::keyring_unlock(keyring = keyringName, password = Sys.getenv('STRATEGUS_KEYRING_PASSWORD')) - } - return(keyringLocked) - } - - keyringName <- jobContext$keyringSettings$keyringName - keyringLocked <- unlockKeyring(keyringName = keyringName) - - ParallelLogger::addDefaultFileLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, 'log.txt')) - ParallelLogger::addDefaultErrorReportLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, 'errorReport.R')) - - options(andromedaTempFolder = file.path(jobContext$moduleExecutionSettings$workFolder, 'andromedaTemp')) + code = { + ############################ + # Generated by strategus - do not modify by hand + ############################ + source("Main.R") + jobContext <- readRDS(jobContextFileName) + + unlockKeyring <- function(keyringName) { + # If the keyring is locked, unlock it, set the value and then re-lock it + keyringLocked <- keyring::keyring_is_locked(keyring = keyringName) + if (keyringLocked) { + keyring::keyring_unlock(keyring = keyringName, password = Sys.getenv("STRATEGUS_KEYRING_PASSWORD")) + } + return(keyringLocked) + } + + keyringName <- jobContext$keyringSettings$keyringName + keyringLocked <- unlockKeyring(keyringName = keyringName) + + ParallelLogger::addDefaultFileLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, "log.txt")) + ParallelLogger::addDefaultErrorReportLogger(file.path(jobContext$moduleExecutionSettings$resultsSubFolder, "errorReport.R")) + + options(andromedaTempFolder = file.path(jobContext$moduleExecutionSettings$workFolder, "andromedaTemp")) options(tempEmulationSchema = jobContext$moduleExecutionSettings$tempEmulationSchema) options(databaseConnectorIntegerAsNumeric = jobContext$moduleExecutionSettings$integerAsNumeric) options(databaseConnectorInteger64AsNumeric = jobContext$moduleExecutionSettings$integer64AsNumeric) - if (Sys.getenv('FORCE_RENV_USE', '') == 'TRUE') { - renv::use(lockfile = 'renv.lock') - } - - # NOTE injected variable isResultsExecution - will look strange outside of Strategus definition - if (isCdmExecution) { - connectionDetails <- Strategus::retrieveConnectionDetails( - connectionDetailsReference = jobContext$moduleExecutionSettings$connectionDetailsReference, - keyringName = keyringName - ) - jobContext$moduleExecutionSettings$connectionDetails <- connectionDetails - } else { - resultsConnectionDetails <- Strategus::retrieveConnectionDetails( - connectionDetailsReference = jobContext$moduleExecutionSettings$resultsConnectionDetailsReference, - keyringName = keyringName - ) - jobContext$moduleExecutionSettings$resultsConnectionDetails <- resultsConnectionDetails - } - if (keyringLocked) { - keyring::keyring_lock(keyring = keyringName) - } - execute(jobContext) - - ParallelLogger::unregisterLogger('DEFAULT_FILE_LOGGER', silent = TRUE) - ParallelLogger::unregisterLogger('DEFAULT_ERRORREPORT_LOGGER', silent = TRUE) - writeLines('done', doneFile) - - }, - tempScriptFile = tempScriptFile, - moduleFolder = moduleFolder, - injectVars = list(doneFile = doneFile, - isCdmExecution = isCdmExecution, - jobContextFileName = jobContextFileName) - ) + if (Sys.getenv("FORCE_RENV_USE", "") == "TRUE") { + renv::use(lockfile = "renv.lock") + } + + # NOTE injected variable isResultsExecution - will look strange outside of Strategus definition + if (isCdmExecution) { + connectionDetails <- Strategus::retrieveConnectionDetails( + connectionDetailsReference = jobContext$moduleExecutionSettings$connectionDetailsReference, + keyringName = keyringName + ) + jobContext$moduleExecutionSettings$connectionDetails <- connectionDetails + } else { + resultsConnectionDetails <- Strategus::retrieveConnectionDetails( + connectionDetailsReference = jobContext$moduleExecutionSettings$resultsConnectionDetailsReference, + keyringName = keyringName + ) + jobContext$moduleExecutionSettings$resultsConnectionDetails <- resultsConnectionDetails + } + if (keyringLocked) { + keyring::keyring_lock(keyring = keyringName) + } + execute(jobContext) + + ParallelLogger::unregisterLogger("DEFAULT_FILE_LOGGER", silent = TRUE) + ParallelLogger::unregisterLogger("DEFAULT_ERRORREPORT_LOGGER", silent = TRUE) + writeLines("done", doneFile) + }, + tempScriptFile = tempScriptFile, + moduleFolder = moduleFolder, + injectVars = list( + doneFile = doneFile, + isCdmExecution = isCdmExecution, + jobContextFileName = jobContextFileName + ) + ) if (!file.exists(doneFile)) { message <- paste( diff --git a/README.md b/README.md index 9ba8b018..39a7921b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Strategus Introduction ============ -Strategus is an R package for coordinating and executing analytics using HADES modules. +Strategus is an R package for coordinating and executing analytics using HADES modules. Please see the [Introduction To Strategus](https://ohdsi.github.io/Strategus/articles/IntroductionToStrategus.html) for more details. Features ======== @@ -21,11 +21,11 @@ Strategus is an R package. System Requirements ============ -Requires R (version 4.0.0 or higher). Installation on Windows requires [RTools](https://cran.r-project.org/bin/windows/Rtools/). Libraries used in Strategus require Java. +Requires R (version 4.2.0 or higher). Installation on Windows requires [RTools](https://cran.r-project.org/bin/windows/Rtools/). Libraries used in Strategus require Java. Strategus requires you to setup your GitHub Personal Access Token as described [here](https://ohdsi.github.io/Hades/rSetup.html#GitHub_Personal_Access_Token) Installation ============= -1. See the instructions [here](https://ohdsi.github.io/Hades/rSetup.html) for configuring your R environment, including RTools and Java. +1. See the instructions [here](https://ohdsi.github.io/Hades/rSetup.html) for configuring your R environment, including RTools, Java and your GitHub Personal Access Token. 2. In R, use the following commands to download and install Strategus: @@ -33,7 +33,15 @@ Installation install.packages("remotes") remotes::install_github("ohdsi/Strategus") ``` - +3. If using Linux, please refer to the [keyring Linux secret service setup](https://r-lib.github.io/keyring/index.html#linux) as this is required to execute studies using Strategus. In addition to those instructions, members of the OHDSI community have shared the following guidance: + +Run in a terminal the following before installing keyring: + +```bash +sudo yum install libsodium libsodium-devel +sudo yum install libsecret libsecret-devel +sudo chmod 777 /home/idies/.config +``` User Documentation ================== diff --git a/docs/404.html b/docs/404.html index e713abb3..09410908 100644 --- a/docs/404.html +++ b/docs/404.html @@ -32,7 +32,7 @@ Strategus - 0.0.6 + 0.1.0 @@ -107,7 +107,7 @@

Page not found (404)