Handle data types for primary keys - character conversion only (#53)

* Handle data types for primary keys - character conversion only * Added chat gipty tests * Added chat gipty tests * changed usage of mittagr * Fixes to tests * Updated news * Package maintenance --------- Co-authored-by: Jamie Gilbert <[email protected]>
OHDSI · May 2, 2024 · 665b71a · 665b71a
1 parent 02a5906
commit 665b71a
Show file tree

Hide file tree

Showing 45 changed files with 193 additions and 85 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: ResultModelManager
 Title: Result Model Manager
-Version: 0.5.6
+Version: 0.5.7
 Authors@R:
     person("Jamie", "Gilbert", , "[email protected]", role = c("aut", "cre"))
 Description: Database data model management utilities for OHDSI packages.

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,9 @@
+# ResultModelManager 0.5.7
+
+Bug fixes:
+
+1. Added type conversion checking on upload of data where columns are characters but interpreted as numeric from reading
+inserted data
 
 # ResultModelManager 0.5.6
 

diff --git a/R/ConnectionHandler.R b/R/ConnectionHandler.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of CemConnector
 #

diff --git a/R/DataMigrationManager.R b/R/DataMigrationManager.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of CohortDiagnostics
 #

diff --git a/R/DataModel.R b/R/DataModel.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of OHdsiSharing
 #
@@ -347,6 +347,10 @@ uploadChunk <- function(chunk, pos, env, specifications, resultsFolder, connecti
 
   # Check if inserting data would violate primary key constraints:
   if (!is.null(env$primaryKeyValuesInDb)) {
+    chunk <- formatChunk(
+      pkValuesInDb = env$primaryKeyValuesInDb,
+      chunk = chunk
+    )
     primaryKeyValuesInChunk <- unique(chunk[env$primaryKey])
     duplicates <-
       dplyr::inner_join(env$primaryKeyValuesInDb,
@@ -791,3 +795,35 @@ loadResultsDataModelSpecifications <- function(filePath) {
   assertSpecificationColumns(colnames(spec))
   return(spec)
 }
+
+
+#' This helper function will convert the data in the
+#' primary key values in the `chunk` which is read from
+#' the csv file to the format of the primary key data
+#' retrieved from the database (`pkValuesInDb`). The assumption made
+#' by this function is that the `pkValuesInDb` reflect the proper data
+#' types while `chunk` is the best guess from the readr
+#' package. In the future, if we adopt strongly-types data.frames
+#' this will no longer be necessary.
+#'
+#' Another assumption of this function is that we're only attempting to
+#' recast to a character data type and not try to handle different type
+#' conversions.
+formatChunk <- function(pkValuesInDb, chunk) {
+  for (columnName in names(pkValuesInDb)) {
+    if (class(pkValuesInDb[[columnName]]) != class(chunk[[columnName]])) {
+      if (class(pkValuesInDb[[columnName]]) == "character") {
+        chunk <- chunk |> dplyr::mutate_at(columnName, as.character)
+      } else {
+        errorMsg <- paste0(
+          columnName,
+          " is of type ",
+          class(pkValuesInDb[[columnName]]),
+          " which cannot be converted between data frames pkValuesInDb and chunk"
+        )
+        stop(errorMsg)
+      }
+    }
+  }
+  return(chunk)
+}
diff --git a/R/PooledConnectionHandler.R b/R/PooledConnectionHandler.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of CohortDiagnostics
 #

diff --git a/R/QueryNamespace.R b/R/QueryNamespace.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of CohortDiagnostics
 #

diff --git a/R/ResultExportManager.R b/R/ResultExportManager.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of CohortDiagnostics
 #

diff --git a/R/ResultModelManager.R b/R/ResultModelManager.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of CohortDiagnostics
 #

diff --git a/R/SchemaGenerator.R b/R/SchemaGenerator.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of CohortDiagnostics
 #

diff --git a/R/Utils.R b/R/Utils.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of CohortDiagnostics
 #

diff --git a/docs/404.html b/docs/404.html
diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html
diff --git a/docs/articles/CreatingMigrations.html b/docs/articles/CreatingMigrations.html
diff --git a/docs/articles/ExampleProject.html b/docs/articles/ExampleProject.html
diff --git a/docs/articles/PackageDesign.html b/docs/articles/PackageDesign.html
diff --git a/docs/articles/UploadFunctionality.html b/docs/articles/UploadFunctionality.html
diff --git a/docs/articles/UsingAnExportManager.html b/docs/articles/UsingAnExportManager.html
diff --git a/docs/articles/UsingConnectionHandlers.html b/docs/articles/UsingConnectionHandlers.html
diff --git a/docs/articles/UsingQueryNamespaces.html b/docs/articles/UsingQueryNamespaces.html
diff --git a/docs/articles/index.html b/docs/articles/index.html
diff --git a/docs/authors.html b/docs/authors.html