Optimizations and New Analytics (#55)

New functionality for making aggregate covariates more efficient.
Adding incremental mode
Removing SQLite
Updated Documentation
Adding Logging.
Validated SQL on all supported DBMS

.github/workflows/R_CMD_check_Hades.yaml

@@ -48,9 +48,20 @@ jobs:
       CDM5_REDSHIFT_PASSWORD: ${{ secrets.CDM5_REDSHIFT_PASSWORD }}
       CDM5_REDSHIFT_SERVER: ${{ secrets.CDM5_REDSHIFT_SERVER }}
       CDM5_REDSHIFT_USER: ${{ secrets.CDM5_REDSHIFT_USER }}
+      CDM_SNOWFLAKE_CDM53_SCHEMA: ${{ secrets.CDM_SNOWFLAKE_CDM53_SCHEMA }}
+      CDM_SNOWFLAKE_OHDSI_SCHEMA: ${{ secrets.CDM_SNOWFLAKE_OHDSI_SCHEMA }}
+      CDM_SNOWFLAKE_PASSWORD: ${{ secrets.CDM_SNOWFLAKE_PASSWORD }}
+      CDM_SNOWFLAKE_CONNECTION_STRING: ${{ secrets.CDM_SNOWFLAKE_CONNECTION_STRING }}
+      CDM_SNOWFLAKE_USER: ${{ secrets.CDM_SNOWFLAKE_USER }}
       CDM5_SPARK_USER: ${{ secrets.CDM5_SPARK_USER }}
       CDM5_SPARK_PASSWORD: ${{ secrets.CDM5_SPARK_PASSWORD }}
       CDM5_SPARK_CONNECTION_STRING: ${{ secrets.CDM5_SPARK_CONNECTION_STRING }}
+      CDM5_SPARK_CDM_SCHEMA: ${{ secrets.CDM5_SPARK_CDM_SCHEMA }}
+      CDM5_SPARK_OHDSI_SCHEMA: ${{ secrets.CDM5_SPARK_OHDSI_SCHEMA }}
+      CDM_BIG_QUERY_CONNECTION_STRING: ${{ secrets.CDM_BIG_QUERY_CONNECTION_STRING }}
+      CDM_BIG_QUERY_KEY_FILE: ${{ secrets.CDM_BIG_QUERY_KEY_FILE }}
+      CDM_BIG_QUERY_CDM_SCHEMA: ${{ secrets.CDM_BIG_QUERY_CDM_SCHEMA }}
+      CDM_BIG_QUERY_OHDSI_SCHEMA: ${{ secrets.CDM_BIG_QUERY_OHDSI_SCHEMA }}
 
     steps:
       - uses: actions/checkout@v3

DESCRIPTION

@@ -1,11 +1,12 @@
 Package: Characterization
 Type: Package
 Title: Characterizations of Cohorts
-Version: 0.2.0
-Date: 2024-04-03
+Version: 1.1.1
+Date: 2024-08-07
 Authors@R: c(
 	person("Jenna", "Reps", , "[email protected]", role = c("aut", "cre")),
-	person("Patrick", "Ryan", , "[email protected]", role = c("aut"))
+	person("Patrick", "Ryan", , "[email protected]", role = c("aut")),
+	person("Chris", "Knoll", , "[email protected]", role = c("aut"))
 	)
 Maintainer: Jenna Reps <[email protected]>
 Description: Various characterizations of target and outcome cohorts.
@@ -17,9 +18,10 @@ Depends:
 Imports:
   Andromeda,
 	DatabaseConnector (>= 6.3.1),
-	FeatureExtraction  (>= 3.5.0),
+	FeatureExtraction  (>= 3.6.0),
 	SqlRender (>= 1.9.0),
 	ParallelLogger (>= 3.0.0),
+	ResultModelManager,
 	checkmate,
 	dplyr,
 	readr, 
@@ -31,16 +33,15 @@ Suggests:
 	kableExtra,
 	knitr,
 	markdown,
-	ResultModelManager,
+	rmarkdown,
 	ShinyAppBuilder,
 	shiny,
 	withr
 Remotes:
 	ohdsi/FeatureExtraction,
 	ohdsi/Eunomia,
 	ohdsi/ResultModelManager,
-	ohdsi/ShinyAppBuilder,
-	ohdsi/DatabaseConnector
+	ohdsi/ShinyAppBuilder
 NeedsCompilation: no
 RoxygenNote: 7.3.1
 Encoding: UTF-8

NAMESPACE

@@ -1,31 +1,25 @@
 # Generated by roxygen2: do not edit by hand
 
-export(computeAggregateCovariateAnalyses)
+export(cleanIncremental)
+export(cleanNonIncremental)
 export(computeDechallengeRechallengeAnalyses)
 export(computeRechallengeFailCaseSeriesAnalyses)
 export(computeTimeToEventAnalyses)
 export(createAggregateCovariateSettings)
 export(createCharacterizationSettings)
 export(createCharacterizationTables)
 export(createDechallengeRechallengeSettings)
+export(createDuringCovariateSettings)
 export(createSqliteDatabase)
 export(createTimeToEventSettings)
-export(exportAggregateCovariateToCsv)
-export(exportDatabaseToCsv)
 export(exportDechallengeRechallengeToCsv)
 export(exportRechallengeFailCaseSeriesToCsv)
 export(exportTimeToEventToCsv)
-export(loadAggregateCovariateAnalyses)
+export(getDbDuringCovariateData)
+export(insertResultsToDatabase)
 export(loadCharacterizationSettings)
-export(loadDechallengeRechallengeAnalyses)
-export(loadRechallengeFailCaseSeriesAnalyses)
-export(loadTimeToEventAnalyses)
 export(runCharacterizationAnalyses)
-export(saveAggregateCovariateAnalyses)
 export(saveCharacterizationSettings)
-export(saveDechallengeRechallengeAnalyses)
-export(saveRechallengeFailCaseSeriesAnalyses)
-export(saveTimeToEventAnalyses)
 export(viewCharacterization)
 importFrom(dplyr,"%>%")
 importFrom(rlang,.data)

NEWS.md

@@ -1,3 +1,38 @@
+Characterization 1.1.1
+======================
+- added tests for all HADES supported dbms
+- updated minCellCount censoring 
+
+Characterization 1.1.0
+======================
+- fixed issues with incremental
+- made the code more modular to enable new characterizations to be added
+- added job optimization code to optimize the distributuion of jobs
+- fixed tests and made minor bug updates
+
+Characterization 1.0.0
+======================
+- Added parallelization in the aggregate covariates analysis
+- Extact all results as csv files 
+- Removed sqlite result creation
+- now using ResultModelManager to upload results into database
+
+Characterization 0.3.1
+======================
+- Removed DatabaseConnector from Remotes in DESCRIPTION. Fixes GitHb issue 38.
+- Added check to covariateSettings input in createAggregateCovariateSettings to error if temporal is T
+- adding during cohort covariate settings
+- added a case covariate settings inputs to aggregate covariates
+- added casePreTargetDuration and casePostTreatmentDuration integer inputs to aggregate covariates
+
+Characterization 0.3.0
+======================
+- Added new outcomeWashoutDays parameter to createAggregateCovariateSettings to remove outcome occurances that are continuations of a prior outcome occurrence
+- Changed the way cohort definition ids are created in createAggregateCovariateSettings to use hash of target id, outcome id and type.  This lets users combine different studies into a single result database.
+- Added database migration capability and created new migrations for the recent updates.
+
+
+
 Characterization 0.2.0
 ======================
 Updated dependency to FeatureExtraction (>= 3.5.0) to support minCharacterizationMean paramater.