diff --git a/DESCRIPTION b/DESCRIPTION index 8184983..3d17f78 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: ResultModelManager Title: Result Model Manager -Version: 0.5.6 +Version: 0.5.7 Authors@R: person("Jamie", "Gilbert", , "gilbert@ohdsi.org", role = c("aut", "cre")) Description: Database data model management utilities for OHDSI packages. diff --git a/NEWS.md b/NEWS.md index b79f504..b72bd84 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +# ResultModelManager 0.5.7 + +Bug fixes: + +1. Added type conversion checking on upload of data where columns are characters but interpreted as numeric from reading +inserted data # ResultModelManager 0.5.6 diff --git a/R/ConnectionHandler.R b/R/ConnectionHandler.R index 8d640f5..f70e7ac 100644 --- a/R/ConnectionHandler.R +++ b/R/ConnectionHandler.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CemConnector # diff --git a/R/DataMigrationManager.R b/R/DataMigrationManager.R index 0aa2e46..f84eadc 100644 --- a/R/DataMigrationManager.R +++ b/R/DataMigrationManager.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CohortDiagnostics # diff --git a/R/DataModel.R b/R/DataModel.R index 4531405..250e4e9 100644 --- a/R/DataModel.R +++ b/R/DataModel.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of OHdsiSharing # @@ -347,6 +347,10 @@ uploadChunk <- function(chunk, pos, env, specifications, resultsFolder, connecti # Check if inserting data would violate primary key constraints: if (!is.null(env$primaryKeyValuesInDb)) { + chunk <- formatChunk( + pkValuesInDb = env$primaryKeyValuesInDb, + chunk = chunk + ) primaryKeyValuesInChunk <- unique(chunk[env$primaryKey]) duplicates <- dplyr::inner_join(env$primaryKeyValuesInDb, @@ -791,3 +795,35 @@ loadResultsDataModelSpecifications <- function(filePath) { assertSpecificationColumns(colnames(spec)) return(spec) } + + +#' This helper function will convert the data in the +#' primary key values in the `chunk` which is read from +#' the csv file to the format of the primary key data +#' retrieved from the database (`pkValuesInDb`). The assumption made +#' by this function is that the `pkValuesInDb` reflect the proper data +#' types while `chunk` is the best guess from the readr +#' package. In the future, if we adopt strongly-types data.frames +#' this will no longer be necessary. +#' +#' Another assumption of this function is that we're only attempting to +#' recast to a character data type and not try to handle different type +#' conversions. +formatChunk <- function(pkValuesInDb, chunk) { + for (columnName in names(pkValuesInDb)) { + if (class(pkValuesInDb[[columnName]]) != class(chunk[[columnName]])) { + if (class(pkValuesInDb[[columnName]]) == "character") { + chunk <- chunk |> dplyr::mutate_at(columnName, as.character) + } else { + errorMsg <- paste0( + columnName, + " is of type ", + class(pkValuesInDb[[columnName]]), + " which cannot be converted between data frames pkValuesInDb and chunk" + ) + stop(errorMsg) + } + } + } + return(chunk) +} diff --git a/R/PooledConnectionHandler.R b/R/PooledConnectionHandler.R index c204330..caff359 100644 --- a/R/PooledConnectionHandler.R +++ b/R/PooledConnectionHandler.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CohortDiagnostics # diff --git a/R/QueryNamespace.R b/R/QueryNamespace.R index 651df14..0d346d3 100644 --- a/R/QueryNamespace.R +++ b/R/QueryNamespace.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CohortDiagnostics # diff --git a/R/ResultExportManager.R b/R/ResultExportManager.R index 8f45044..1d47b35 100644 --- a/R/ResultExportManager.R +++ b/R/ResultExportManager.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CohortDiagnostics # diff --git a/R/ResultModelManager.R b/R/ResultModelManager.R index 7863103..ef9661e 100644 --- a/R/ResultModelManager.R +++ b/R/ResultModelManager.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CohortDiagnostics # diff --git a/R/SchemaGenerator.R b/R/SchemaGenerator.R index 14b9c46..11232ac 100644 --- a/R/SchemaGenerator.R +++ b/R/SchemaGenerator.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CohortDiagnostics # diff --git a/R/Utils.R b/R/Utils.R index c0ade61..8b0edec 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CohortDiagnostics # diff --git a/docs/404.html b/docs/404.html index 96e4e04..8b858dc 100644 --- a/docs/404.html +++ b/docs/404.html @@ -32,7 +32,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html index 85cf394..23b1275 100644 --- a/docs/LICENSE-text.html +++ b/docs/LICENSE-text.html @@ -17,7 +17,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/articles/CreatingMigrations.html b/docs/articles/CreatingMigrations.html index eb192e9..c5013aa 100644 --- a/docs/articles/CreatingMigrations.html +++ b/docs/articles/CreatingMigrations.html @@ -33,7 +33,7 @@ ResultModelManager - 0.5.6 + 0.5.7 @@ -97,7 +97,7 @@

Creating Migrations

James P. Gilbert

-

2023-12-18

+

2024-05-02

diff --git a/docs/articles/ExampleProject.html b/docs/articles/ExampleProject.html index a87109b..2c51e7d 100644 --- a/docs/articles/ExampleProject.html +++ b/docs/articles/ExampleProject.html @@ -33,7 +33,7 @@ ResultModelManager - 0.5.6 + 0.5.7 @@ -97,7 +97,7 @@

Example package results spec

James P. Gilbert

-

2023-12-18

+

2024-05-02

@@ -253,14 +253,8 @@

Creating a results database schema
 # note - the table prefix and schema parameters are not neeeded when we do this
 qns$executeSql(sql)
-
## 
-  |                                                                            
-  |                                                                      |   0%
-  |                                                                            
-  |===================================                                   |  50%
-  |                                                                            
-  |======================================================================| 100%
-
## Executing SQL took 0.0192 secs
+
##   |                                                                              |                                                                      |   0%  |                                                                              |===================================                                   |  50%  |                                                                              |======================================================================| 100%
+
## Executing SQL took 0.0276 secs

Alternatively, we can just use DatabaseConnector functions directly.

diff --git a/docs/articles/PackageDesign.html b/docs/articles/PackageDesign.html
index 62ef259..0e3c054 100644
--- a/docs/articles/PackageDesign.html
+++ b/docs/articles/PackageDesign.html
@@ -33,7 +33,7 @@
       
       
         ResultModelManager
-        0.5.6
+        0.5.7
@@ -97,7 +97,7 @@

Package Design

James P. Gilbert

-

2023-12-18

+

2024-05-02

diff --git a/docs/articles/UploadFunctionality.html b/docs/articles/UploadFunctionality.html index f3308b6..36486dd 100644 --- a/docs/articles/UploadFunctionality.html +++ b/docs/articles/UploadFunctionality.html @@ -33,7 +33,7 @@ ResultModelManager - 0.5.6 + 0.5.7 @@ -97,7 +97,7 @@

Upload Functionality

James P. Gilbert

-

2023-12-18

+

2024-05-02

diff --git a/docs/articles/UsingAnExportManager.html b/docs/articles/UsingAnExportManager.html index 832bfc0..2615717 100644 --- a/docs/articles/UsingAnExportManager.html +++ b/docs/articles/UsingAnExportManager.html @@ -33,7 +33,7 @@ ResultModelManager - 0.5.6 + 0.5.7 @@ -97,7 +97,7 @@

Using An Export Manager

James P. Gilbert

-

2023-12-18

+

2024-05-02

@@ -211,7 +211,7 @@

Performing R operationslibrary(rateratio.test) transformation <- function(rows, pos) { - rrResult <- rateratio.test( + rrResult <- rateratio.test( x = c(row$target_count, row$comparator_count), n = c(row$target_time, row$comparator_time), RR = 1, @@ -241,7 +241,7 @@

Exporting an Andromeda result in export in to the R session before saving to disk. Instead, it is best practice to use batch operations as follows

-andr <- Andromeda::andromeda()
+andr <- Andromeda::andromeda()
 andr$my_andromeda_table <- data.frame(covariate_id = 1:1e4, value = stats::runif(1e4))
 
 first <- TRUE
@@ -252,7 +252,7 @@ 
Exporting an Andromeda result in
   return(invisible(NULL))
 }
 
-Andromeda::batchApply(andr$my_andromeda_table, writeBatch)
+Andromeda::batchApply(andr$my_andromeda_table, writeBatch)
diff --git a/docs/articles/UsingConnectionHandlers.html b/docs/articles/UsingConnectionHandlers.html index e867d1a..f5b3e01 100644 --- a/docs/articles/UsingConnectionHandlers.html +++ b/docs/articles/UsingConnectionHandlers.html @@ -33,7 +33,7 @@ ResultModelManager - 0.5.6 + 0.5.7
@@ -97,7 +97,7 @@

Using Connection Handlers

James P. Gilbert

-

2023-12-18

+

2024-05-02

diff --git a/docs/articles/UsingQueryNamespaces.html b/docs/articles/UsingQueryNamespaces.html index 23e50f4..63b4a24 100644 --- a/docs/articles/UsingQueryNamespaces.html +++ b/docs/articles/UsingQueryNamespaces.html @@ -33,7 +33,7 @@ ResultModelManager - 0.5.6 + 0.5.7 @@ -97,7 +97,7 @@

Using Query Namespaces

James P. Gilbert

-

2023-12-18

+

2024-05-02

@@ -159,12 +159,8 @@

Basic usagesql <- generateSqlSchema(schemaDefinition = tableSpecification) # note - the table prefix and schema parameters are not neeeded qns$executeSql(sql) -
## 
-  |                                                                            
-  |                                                                      |   0%
-  |                                                                            
-  |======================================================================| 100%
-
## Executing SQL took 0.0151 secs
+
##   |                                                                              |                                                                      |   0%  |                                                                              |======================================================================| 100%
+
## Executing SQL took 0.0202 secs

We can then query the table with sql that automatically replaces the table names:

diff --git a/docs/articles/index.html b/docs/articles/index.html
index 912fbb0..b13f142 100644
--- a/docs/articles/index.html
+++ b/docs/articles/index.html
@@ -17,7 +17,7 @@
       
       
         ResultModelManager
-        0.5.6
+        0.5.7
diff --git a/docs/authors.html b/docs/authors.html index 96c0844..778ef8d 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -17,7 +17,7 @@ ResultModelManager - 0.5.6 + 0.5.7 @@ -86,15 +86,15 @@

Citation

-

Gilbert J (2023). -ResultModelManager: Result Model Manager (RMM) for OHDSI packages. -R package version 0.5.6. +

Gilbert J (2024). +ResultModelManager: Result Model Manager. +R package version 0.5.7. 

@Manual{,
-  title = {ResultModelManager: Result Model Manager (RMM) for OHDSI packages},
+  title = {ResultModelManager: Result Model Manager},
   author = {Jamie Gilbert},
-  year = {2023},
-  note = {R package version 0.5.6},
+  year = {2024},
+  note = {R package version 0.5.7},
 }
diff --git a/docs/index.html b/docs/index.html index d473049..73da5f5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -5,13 +5,13 @@ -Result Model Manager (RMM) for OHDSI packages • ResultModelManager +Result Model Manager • ResultModelManager - + ResultModelManager: Result Model Manager (RMM) for OHDSI packages — ResultModelManager-package • ResultModelManagerResultModelManager: Result Model Manager — ResultModelManager-package • ResultModelManager @@ -17,7 +17,7 @@ ResultModelManager - 0.5.6 + 0.5.7 @@ -68,7 +68,7 @@
-

ResultModelManager: Result Model Manager (RMM) for OHDSI packages

+

ResultModelManager: Result Model Manager

diff --git a/docs/reference/createQueryNamespace.html b/docs/reference/createQueryNamespace.html index 3cfa8ba..9a63e87 100644 --- a/docs/reference/createQueryNamespace.html +++ b/docs/reference/createQueryNamespace.html @@ -19,7 +19,7 @@ ResultModelManager - 0.5.6 + 0.5.7
diff --git a/docs/reference/createResultExportManager.html b/docs/reference/createResultExportManager.html index 0c874c5..32286c8 100644 --- a/docs/reference/createResultExportManager.html +++ b/docs/reference/createResultExportManager.html @@ -23,7 +23,7 @@ ResultModelManager - 0.5.6 + 0.5.7
diff --git a/docs/reference/deleteAllRowsForDatabaseId.html b/docs/reference/deleteAllRowsForDatabaseId.html index 21be55d..26b022b 100644 --- a/docs/reference/deleteAllRowsForDatabaseId.html +++ b/docs/reference/deleteAllRowsForDatabaseId.html @@ -17,7 +17,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/reference/deleteAllRowsForPrimaryKey.html b/docs/reference/deleteAllRowsForPrimaryKey.html index c998203..33a9e37 100644 --- a/docs/reference/deleteAllRowsForPrimaryKey.html +++ b/docs/reference/deleteAllRowsForPrimaryKey.html @@ -17,7 +17,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/reference/generateSqlSchema.html b/docs/reference/generateSqlSchema.html index 3cda85b..581d60e 100644 --- a/docs/reference/generateSqlSchema.html +++ b/docs/reference/generateSqlSchema.html @@ -17,7 +17,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/reference/grantTablePermissions.html b/docs/reference/grantTablePermissions.html index a9f4b4d..2ca2d47 100644 --- a/docs/reference/grantTablePermissions.html +++ b/docs/reference/grantTablePermissions.html @@ -19,7 +19,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/reference/index.html b/docs/reference/index.html index 0121632..9b87553 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -17,7 +17,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/reference/loadResultsDataModelSpecifications.html b/docs/reference/loadResultsDataModelSpecifications.html index d5615fc..ccec443 100644 --- a/docs/reference/loadResultsDataModelSpecifications.html +++ b/docs/reference/loadResultsDataModelSpecifications.html @@ -17,7 +17,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/reference/unzipResults.html b/docs/reference/unzipResults.html index 6b291b0..1f3e7df 100644 --- a/docs/reference/unzipResults.html +++ b/docs/reference/unzipResults.html @@ -21,7 +21,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/docs/reference/uploadResults.html b/docs/reference/uploadResults.html index f55007e..527a827 100644 --- a/docs/reference/uploadResults.html +++ b/docs/reference/uploadResults.html @@ -20,7 +20,7 @@ ResultModelManager - 0.5.6 + 0.5.7 diff --git a/extras/ResultModelManager.pdf b/extras/ResultModelManager.pdf index 8b7da07..3cfb9e2 100644 Binary files a/extras/ResultModelManager.pdf and b/extras/ResultModelManager.pdf differ diff --git a/tests/testthat/helper-packageMaintainer.R b/tests/testthat/helper-packageMaintainer.R index 125578a..9a4bb67 100644 --- a/tests/testthat/helper-packageMaintainer.R +++ b/tests/testthat/helper-packageMaintainer.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2024 Observational Health Data Sciences and Informatics # # This file is part of CohortDiagnostics # diff --git a/tests/testthat/test-DataModelFunctions.R b/tests/testthat/test-DataModelFunctions.R index ab6658a..dc69771 100644 --- a/tests/testthat/test-DataModelFunctions.R +++ b/tests/testthat/test-DataModelFunctions.R @@ -26,3 +26,56 @@ test_that("Bad model format", { checkmate::expect_character(data$charField) checkmate::expect_date(data$dateField) }) + + +# Create test data +pkValuesInDb <- data.frame( + id = c(1, 2, 3), + name = c("Alice", "Bob", "Charlie") +) + +chunk <- data.frame( + id = c(1, 2, 3), + name = c("Alice", "Bob", "Charlie") +) + +# Tests for when types match +test_that("formatChunk converts data types correctly", { + expect_identical( + formatChunk(pkValuesInDb, chunk), + chunk, + info = "Data frames should remain unchanged when types match." + ) +}) + +# Tests for when types need conversion +char <- data.frame( + id = c(1, 2, 3), + name = c("Alice", "Bob", "Charlie") +) + +pkValuesInDbChar <- data.frame( + id = as.character(c(1, 2, 3)), + name = c("Alice", "Bob", "Charlie") +) + +test_that("formatChunk converts non-matching types to character", { + expect_identical( + formatChunk(pkValuesInDbChar, char)$id, + as.character(char$id), + info = "Numeric columns in chunk should be converted to character." + ) +}) + +# Test for error when types cannot be converted +chunk_factor <- data.frame( + id = as.character(c(1, 2, 3)), + name = c("Alice", "Bob", "Charlie") +) + +test_that("formatChunk throws error for incompatible types", { + expect_error( + formatChunk(pkValuesInDb, chunk_factor), + "id is of type numeric which cannot be converted between data frames pkValuesInDb and chunk" + ) +})