From 57d580e0577cb31dbe554609fd490caadf415e84 Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Thu, 24 Oct 2024 09:51:11 -0400 Subject: [PATCH 01/12] add 2 docs pages --- _pkgdown.yml | 4 + docs/404.html | 6 ++ docs/LICENSE-text.html | 6 ++ docs/articles/AddNewCheck.html | 8 +- docs/articles/CheckStatusDefinitions.html | 8 +- docs/articles/CheckTypeDescriptions.html | 8 +- docs/articles/DataQualityDashboard.html | 8 +- docs/articles/DqdForCohorts.html | 8 +- docs/articles/SqlOnly.html | 8 +- docs/articles/Thresholds.html | 8 +- docs/articles/checkIndex.html | 12 ++- docs/articles/checks/cdmDatatype.html | 8 +- docs/articles/checks/cdmField.html | 8 +- docs/articles/checks/cdmTable.html | 8 +- docs/articles/checks/fkClass.html | 8 +- docs/articles/checks/fkDomain.html | 8 +- docs/articles/checks/isForeignKey.html | 8 +- docs/articles/checks/isPrimaryKey.html | 8 +- docs/articles/checks/isRequired.html | 8 +- .../checks/isStandardValidConcept.html | 8 +- .../measureConditionEraCompleteness.html | 8 +- .../checks/measurePersonCompleteness.html | 8 +- .../checks/measureValueCompleteness.html | 8 +- docs/articles/checks/plausibleAfterBirth.html | 8 +- .../articles/checks/plausibleBeforeDeath.html | 8 +- .../checks/plausibleGenderUseDescendants.html | 87 ++++++++++++++++--- .../checks/plausibleStartBeforeEnd.html | 8 +- .../checks/plausibleTemporalAfter.html | 8 +- .../checks/plausibleUnitConceptIds.html | 81 +++++++++++++++-- docs/articles/checks/plausibleValueHigh.html | 8 +- docs/articles/checks/plausibleValueLow.html | 8 +- .../sourceConceptRecordCompleteness.html | 8 +- .../checks/sourceValueCompleteness.html | 8 +- .../standardConceptRecordCompleteness.html | 8 +- docs/articles/checks/withinVisitDates.html | 8 +- docs/articles/index.html | 6 ++ docs/authors.html | 6 ++ docs/index.html | 6 ++ docs/news/index.html | 6 ++ docs/pkgdown.yml | 2 +- .../reference/convertJsonResultsFileCase.html | 6 ++ docs/reference/dot-applyNotApplicable.html | 8 +- .../dot-calculateNotApplicableStatus.html | 8 +- docs/reference/dot-containsNAchecks.html | 8 +- docs/reference/dot-evaluateThresholds.html | 6 ++ docs/reference/dot-getCheckId.html | 6 ++ docs/reference/dot-hasNAchecks.html | 8 +- docs/reference/dot-processCheck.html | 6 ++ docs/reference/dot-recordResult.html | 6 ++ docs/reference/dot-runCheck.html | 6 ++ docs/reference/dot-summarizeResults.html | 6 ++ docs/reference/dot-writeResultsToCsv.html | 6 ++ docs/reference/dot-writeResultsToJson.html | 6 ++ docs/reference/dot-writeResultsToTable.html | 6 ++ docs/reference/executeDqChecks.html | 6 ++ docs/reference/index.html | 6 ++ docs/reference/listDqChecks.html | 6 ++ docs/reference/reEvaluateThresholds.html | 6 ++ docs/reference/viewDqDashboard.html | 6 ++ docs/reference/writeDBResultsToJson.html | 6 ++ docs/reference/writeJsonResultsToCsv.html | 6 ++ docs/reference/writeJsonResultsToTable.html | 6 ++ vignettes/checkIndex.Rmd | 4 +- .../checks/plausibleGenderUseDescendants.Rmd | 39 +++++++-- vignettes/checks/plausibleUnitConceptIds.Rmd | 35 ++++++-- 65 files changed, 595 insertions(+), 77 deletions(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index 6d2283a1..beaa8dc6 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -94,6 +94,10 @@ navbar: href: articles/checks/withinVisitDates.html - text: measureConditionEraCompleteness href: articles/checks/measureConditionEraCompleteness.html + - text: plausibleUnitConceptIds + href: articles/checks/plausibleUnitConceptIds.html + - text: plausibleGenderUseDescendants + href: articles/checks/plausibleGenderUseDescendants.html hades: text: hadesLogo href: https://ohdsi.github.io/Hades diff --git a/docs/404.html b/docs/404.html index 95c2863b..fa303700 100644 --- a/docs/404.html +++ b/docs/404.html @@ -150,6 +150,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html index f3e311f0..d75a8838 100644 --- a/docs/LICENSE-text.html +++ b/docs/LICENSE-text.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/articles/AddNewCheck.html b/docs/articles/AddNewCheck.html index b35776e2..6b3ed4b9 100644 --- a/docs/articles/AddNewCheck.html +++ b/docs/articles/AddNewCheck.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -183,7 +189,7 @@

    Add a New Data Quality Check

    Don Torok

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/AddNewCheck.rmd diff --git a/docs/articles/CheckStatusDefinitions.html b/docs/articles/CheckStatusDefinitions.html index ab4023cf..204e2a56 100644 --- a/docs/articles/CheckStatusDefinitions.html +++ b/docs/articles/CheckStatusDefinitions.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    Check Status Definitions

    Dmitry Ilyn, Maxim Moinat

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/CheckStatusDefinitions.rmd diff --git a/docs/articles/CheckTypeDescriptions.html b/docs/articles/CheckTypeDescriptions.html index f5e7016e..65c42ac7 100644 --- a/docs/articles/CheckTypeDescriptions.html +++ b/docs/articles/CheckTypeDescriptions.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    Data Quality Check Type Definitions

    Clair Blacketer

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/CheckTypeDescriptions.rmd diff --git a/docs/articles/DataQualityDashboard.html b/docs/articles/DataQualityDashboard.html index f61d5f7d..4efa6950 100644 --- a/docs/articles/DataQualityDashboard.html +++ b/docs/articles/DataQualityDashboard.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    Getting Started

    Clair Blacketer

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/DataQualityDashboard.rmd diff --git a/docs/articles/DqdForCohorts.html b/docs/articles/DqdForCohorts.html index ef099b37..fe75ccbe 100644 --- a/docs/articles/DqdForCohorts.html +++ b/docs/articles/DqdForCohorts.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    Running the DQD on a Cohort

    Clair Blacketer

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/DqdForCohorts.rmd diff --git a/docs/articles/SqlOnly.html b/docs/articles/SqlOnly.html index 3aa3bf25..6d3e21da 100644 --- a/docs/articles/SqlOnly.html +++ b/docs/articles/SqlOnly.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    Running the DQD in SqlOnly mode

    Maxim Moinat

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/SqlOnly.rmd diff --git a/docs/articles/Thresholds.html b/docs/articles/Thresholds.html index 933f72c5..bfa30bf5 100644 --- a/docs/articles/Thresholds.html +++ b/docs/articles/Thresholds.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    Failure Thresholds and How to Change Them

    Clair Blacketer

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/Thresholds.rmd diff --git a/docs/articles/checkIndex.html b/docs/articles/checkIndex.html index 41fdcdf1..18d405dc 100644 --- a/docs/articles/checkIndex.html +++ b/docs/articles/checkIndex.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    Index

    Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checkIndex.Rmd @@ -251,8 +257,8 @@

    Checks
  • plausibleAfterBirth
  • plausibleBeforeDeath
  • plausibleStartBeforeEnd
    • -
  • plausibleGenderUseDescendants (PAGE UNDER CONSTRUCTION)
    • -
  • plausibleUnitConceptIds (PAGE UNDER CONSTRUCTION)
    • +
  • plausibleGenderUseDescendants
    • +
  • plausibleUnitConceptIds
    • diff --git a/docs/articles/checks/cdmDatatype.html b/docs/articles/checks/cdmDatatype.html index 8e292b75..1cda7b43 100644 --- a/docs/articles/checks/cdmDatatype.html +++ b/docs/articles/checks/cdmDatatype.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +

  • @@ -184,7 +190,7 @@

    cdmDatatype

    Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/cdmDatatype.Rmd diff --git a/docs/articles/checks/cdmField.html b/docs/articles/checks/cdmField.html index d12a1ac0..ef18522b 100644 --- a/docs/articles/checks/cdmField.html +++ b/docs/articles/checks/cdmField.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    cdmField

    Heidi Schmidt, Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/cdmField.Rmd diff --git a/docs/articles/checks/cdmTable.html b/docs/articles/checks/cdmTable.html index e0aeeb50..4eef06a6 100644 --- a/docs/articles/checks/cdmTable.html +++ b/docs/articles/checks/cdmTable.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    cdmTable

    John Gresh, Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/cdmTable.Rmd diff --git a/docs/articles/checks/fkClass.html b/docs/articles/checks/fkClass.html index fb148775..2910051a 100644 --- a/docs/articles/checks/fkClass.html +++ b/docs/articles/checks/fkClass.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    fkClass

    Clair Blacketer, Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/fkClass.Rmd diff --git a/docs/articles/checks/fkDomain.html b/docs/articles/checks/fkDomain.html index a7e873ec..2d14b4bb 100644 --- a/docs/articles/checks/fkDomain.html +++ b/docs/articles/checks/fkDomain.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    fkDomain

    Clair Blacketer, Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/fkDomain.Rmd diff --git a/docs/articles/checks/isForeignKey.html b/docs/articles/checks/isForeignKey.html index 302705ce..18f9e797 100644 --- a/docs/articles/checks/isForeignKey.html +++ b/docs/articles/checks/isForeignKey.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    isForeignKey

    Dmytry Dymshyts, Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/isForeignKey.Rmd diff --git a/docs/articles/checks/isPrimaryKey.html b/docs/articles/checks/isPrimaryKey.html index cbb1714d..b93b248b 100644 --- a/docs/articles/checks/isPrimaryKey.html +++ b/docs/articles/checks/isPrimaryKey.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    isPrimaryKey

    John Gresh, Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/isPrimaryKey.Rmd diff --git a/docs/articles/checks/isRequired.html b/docs/articles/checks/isRequired.html index a69f5776..aba5e96c 100644 --- a/docs/articles/checks/isRequired.html +++ b/docs/articles/checks/isRequired.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    isRequired

    Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/isRequired.Rmd diff --git a/docs/articles/checks/isStandardValidConcept.html b/docs/articles/checks/isStandardValidConcept.html index c945fb74..2e1a3789 100644 --- a/docs/articles/checks/isStandardValidConcept.html +++ b/docs/articles/checks/isStandardValidConcept.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    isStandardValidConcept

    Stephanie Hong, Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/isStandardValidConcept.Rmd diff --git a/docs/articles/checks/measureConditionEraCompleteness.html b/docs/articles/checks/measureConditionEraCompleteness.html index b6ce9b11..eb988271 100644 --- a/docs/articles/checks/measureConditionEraCompleteness.html +++ b/docs/articles/checks/measureConditionEraCompleteness.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    measureConditionEraCompleteness

    Maxim Moinat

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/measureConditionEraCompleteness.Rmd diff --git a/docs/articles/checks/measurePersonCompleteness.html b/docs/articles/checks/measurePersonCompleteness.html index c59f7f48..ad7b0a67 100644 --- a/docs/articles/checks/measurePersonCompleteness.html +++ b/docs/articles/checks/measurePersonCompleteness.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    measurePersonCompleteness

    Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/measurePersonCompleteness.Rmd diff --git a/docs/articles/checks/measureValueCompleteness.html b/docs/articles/checks/measureValueCompleteness.html index dd425880..a6140dea 100644 --- a/docs/articles/checks/measureValueCompleteness.html +++ b/docs/articles/checks/measureValueCompleteness.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    measureValueCompleteness

    Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/measureValueCompleteness.Rmd diff --git a/docs/articles/checks/plausibleAfterBirth.html b/docs/articles/checks/plausibleAfterBirth.html index f9d75774..4f27a852 100644 --- a/docs/articles/checks/plausibleAfterBirth.html +++ b/docs/articles/checks/plausibleAfterBirth.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    plausibleAfterBirth

    Maxim Moinat, Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/plausibleAfterBirth.Rmd diff --git a/docs/articles/checks/plausibleBeforeDeath.html b/docs/articles/checks/plausibleBeforeDeath.html index fd3cd17a..ae82f8e7 100644 --- a/docs/articles/checks/plausibleBeforeDeath.html +++ b/docs/articles/checks/plausibleBeforeDeath.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    plausibleBeforeDeath

    Maxim Moinat

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/plausibleBeforeDeath.Rmd diff --git a/docs/articles/checks/plausibleGenderUseDescendants.html b/docs/articles/checks/plausibleGenderUseDescendants.html index ca4f7198..985ffb09 100644 --- a/docs/articles/checks/plausibleGenderUseDescendants.html +++ b/docs/articles/checks/plausibleGenderUseDescendants.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -181,9 +187,10 @@

    plausibleGender

    -

    +

    Katy Sadowski +and Dmytry Dymshyts

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/plausibleGenderUseDescendants.Rmd @@ -195,46 +202,102 @@

    2024-07-12

    Summary

    -

    Level: CONCEPT
    Context: Validation
    Category: Plausibility
    Subcategory: Atemporal
    Severity:

    +

    Level: CONCEPT
    Context: Validation
    Category: Plausibility
    Subcategory: Atemporal
    Severity: Characterization ✔

    Description

    -

    For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records -associated with patients with an implausible gender (correct gender = -@plausibleGender).

    +

    For descendants of CONCEPT_ID conceptId (conceptName), the number and +percent of records associated with patients with an implausible gender +(correct gender = plausibleGenderUseDescendants).

    Definition

    +

    This check will count the number of records for which the person’s +gender is implausible given the concept for the record. For a given +gender-specific concept (e.g., prostate cancer) and its descendants, the +check will identify records for which the associated person has an +implausible gender (e.g., a female with prostate cancer). There are +currently 4 instances of this check - female condition concepts; male +condition concepts; female procedure concepts; and male procedure +concepts.

    • -Numerator:
      • +Numerator: The number of rows with a gender-specific +concept whose associated person has an implausible gender.
    • -Denominator:
      • +Denominator: The number of rows with a gender-specific +concept.
    • -Related CDM Convention(s):
      • +Related CDM Convention(s): https://ohdsi.github.io/Themis/populate_gender_concept_id.html +
    • -CDM Fields/Tables:
      • +CDM Fields/Tables: CONDITION_OCURRENCE, +PROCEDURE_OCCURRENCE +
    • -Default Threshold Value:
      • +Default Threshold Value: 5%

    User Guidance

    +

    A failure of this check indicates one of the following scenarios:

    +
      +
    • The person’s gender is wrong
      • +
    • The gender-specific concept is wrong
      • +
    • The person changed genders and the concept was plausible at the time +it was recorded
      • +

    Violated rows query

    -
    +
    SELECT
+  cdmTable.@cdmFieldName,
+  cdmTable.@sourceConceptIdField, -- x_source_concept_id for the table of interest (condition_occurrence or procedure_occurrence)
+  cdmTable.@sourceValueField, -- x_source_value for the table of interest
+  COUNT(*)
+FROM @cdmDatabaseSchema.@cdmTableName cdmTable
+  JOIN @cdmDatabaseSchema.person p ON cdmTable.person_id = p.person_id
+  JOIN @vocabDatabaseSchema.concept_ancestor ca ON ca.descendant_concept_id = cdmTable.@cdmFieldName
+WHERE ca.ancestor_concept_id IN (@conceptId)
+  AND p.gender_concept_id <> {@plausibleGenderUseDescendants == 'Male'} ? {8507} : {8532} 
+GROUP BY 1,2,3
    +

    The above query should help to identify if a mapping issue is the +cause of the failure. If the source value and source concept ID are +correctly mapped to a standard concept, then the issue may be that the +person has the incorrect gender, or that the finding is a true data +anomaly. Examples of true anomalies include:

    +
      +
    • Occasional stray code (e.g., due to typo in EHR). This gives an idea +how noisy the data is.
      • +
    • Newborn codes recorded in the mother’s record (e.g., +circumcision).
      • +
    • Gender reassignment procedures (e.g., penectomy and prostatectomy in +patients with acquired (female) gender). NOTE that this +is technically a violation of the OMOP CDM specification, since the CDM +actually requires that the gender_concept_id represents the +person’s sex at birth. For more information on this convention, see https://ohdsi.github.io/Themis/populate_gender_concept_id.html +
      • +

    ETL Developers

    +

    Concept mapping issues must be fixed. Ensure as well that source +codes are being correctly extracted from the source data. If the CDM +accurately represents the source data, then remaining failures should be +documented for users of the CDM.

    Data Users

    +

    Persons with implausible gender should not be included in analysis +unless it can be confirmed with the data provider that these +represent cases of gender reassignment, and your analysis does not +assume that the gender_concept_id represents sex at +birth.

    diff --git a/docs/articles/checks/plausibleStartBeforeEnd.html b/docs/articles/checks/plausibleStartBeforeEnd.html index ff0de203..7d2aa8c8 100644 --- a/docs/articles/checks/plausibleStartBeforeEnd.html +++ b/docs/articles/checks/plausibleStartBeforeEnd.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    plausibleStartBeforeEnd

    Maxim Moinat

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/plausibleStartBeforeEnd.Rmd diff --git a/docs/articles/checks/plausibleTemporalAfter.html b/docs/articles/checks/plausibleTemporalAfter.html index bd4e87dc..135c6baa 100644 --- a/docs/articles/checks/plausibleTemporalAfter.html +++ b/docs/articles/checks/plausibleTemporalAfter.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -183,7 +189,7 @@

    plausibleTemporalAfter

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/plausibleTemporalAfter.Rmd diff --git a/docs/articles/checks/plausibleUnitConceptIds.html b/docs/articles/checks/plausibleUnitConceptIds.html index 21a18f54..72e249dd 100644 --- a/docs/articles/checks/plausibleUnitConceptIds.html +++ b/docs/articles/checks/plausibleUnitConceptIds.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -181,9 +187,10 @@

    plausibleUnitConceptIds

    -

    +

    Katy +Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/plausibleUnitConceptIds.Rmd @@ -195,7 +202,7 @@

    2024-07-12

    Summary

    -

    Level: CONCEPT
    Context: Verification
    Category: Plausibility
    Subcategory: Atemporal
    Severity:

    +

    Level: CONCEPT
    Context: Verification
    Category: Plausibility
    Subcategory: Atemporal
    Severity: Characterization ✔

    Description @@ -208,32 +215,88 @@

    Definition

    User Guidance

    +

    A failure of this check indicates one of the following:

    +
      +
    • A measurement record has the incorrect unit
      • +
    • A measurement record has no unit when it should have one
      • +
    • A measurement record has a unit when it should not have one
      • +
    • The list of plausible unit concept IDs in the threshold file is +incomplete (please report this as a DataQualityDashboard bug!)
      • +
    +

    The above issues could either be due to incorrect data in the source +system or incorrect mapping of the unit concept IDs in the ETL +process.

    Violated rows query

    -
    +
    SELECT 
+    m.unit_concept_id,
+  m.unit_source_concept_id,
+  m.unit_source_value,
+  COUNT(*)
+FROM @cdmDatabaseSchema.@cdmTableName m
+WHERE m.@cdmFieldName = @conceptId
+    AND COALESCE (m.unit_concept_id, -1) NOT IN (@plausibleUnitConceptIds) -- '-1' stands for the cases when unit_concept_id is null
+    AND m.value_as_number IS NOT NULL 
+GROUP BY 1,2,3
    +

    Inspect the output of the violated rows query to identify the root +cause of the issue. If the unit_source_value and/or +unit_source_concept_id are populated, check them against +the list of plausible unit concept IDs to understand if they should have +been mapped to one of the plausible standard concepts. If the +unit_source_value is NULL but the list of plausible unit +concept IDs includes -1, then you may need to check your source data to +undersand whether or not a unit is available in the source.

    ETL Developers

    +

    Ensure that all units available in the source data are being pulled +into the CDM and mapped correctly to a standard concept ID. If a unit is +available in the source and is being correctly populated & mapped in +your ETL but is not present on the list of plausible unit +concept IDs, you should verify whether or not the unit is actually +plausible - you may need to consult a clinician to do so. If the unit is +plausible for the given measurement, please report this as a +DataQualityDashboard bug here: https://github.com/OHDSI/DataQualityDashboard/issues. If +the unit is not plausible, do not change it! Instead, you should +document the issue for users of the CDM and discuss with your data +provider how to handle the data.

    Data Users

    +

    It is generally recommended not to use measurements with implausible +units in analyses as it is impossible to determine whether the unit is +wrong; the value is wrong; and/or the measurement code is wrong in the +source data. If a measurement is missing a unit_concept_id +due to an ETL issue, and the unit_source_value or +unit_source_concept_id is available, you can utilize these +values to perform your analysis. If unit_source_value and +unit_source_concept_id are missing, you may consider +consulting with your data provider as to if and when you may be able to +infer what the missing unit should be.

    diff --git a/docs/articles/checks/plausibleValueHigh.html b/docs/articles/checks/plausibleValueHigh.html index fe3f399b..d63d09d0 100644 --- a/docs/articles/checks/plausibleValueHigh.html +++ b/docs/articles/checks/plausibleValueHigh.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    plausibleValueHigh

    Dymytry Dymshyts

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/plausibleValueHigh.Rmd diff --git a/docs/articles/checks/plausibleValueLow.html b/docs/articles/checks/plausibleValueLow.html index 522ff118..1e708978 100644 --- a/docs/articles/checks/plausibleValueLow.html +++ b/docs/articles/checks/plausibleValueLow.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    plausibleValueLow

    Dymytry Dymshyts

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/plausibleValueLow.Rmd diff --git a/docs/articles/checks/sourceConceptRecordCompleteness.html b/docs/articles/checks/sourceConceptRecordCompleteness.html index d152f815..1f217e6d 100644 --- a/docs/articles/checks/sourceConceptRecordCompleteness.html +++ b/docs/articles/checks/sourceConceptRecordCompleteness.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    sourceConceptRecordCompleteness

    Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/sourceConceptRecordCompleteness.Rmd diff --git a/docs/articles/checks/sourceValueCompleteness.html b/docs/articles/checks/sourceValueCompleteness.html index dd3789cc..2e2e4329 100644 --- a/docs/articles/checks/sourceValueCompleteness.html +++ b/docs/articles/checks/sourceValueCompleteness.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    sourceValueCompleteness

    Jared Houghtaling, Clair Blacketer

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/sourceValueCompleteness.Rmd diff --git a/docs/articles/checks/standardConceptRecordCompleteness.html b/docs/articles/checks/standardConceptRecordCompleteness.html index 5f2d923c..67182b8b 100644 --- a/docs/articles/checks/standardConceptRecordCompleteness.html +++ b/docs/articles/checks/standardConceptRecordCompleteness.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    standardConceptRecordCompleteness

    Katy Sadowski

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/standardConceptRecordCompleteness.Rmd diff --git a/docs/articles/checks/withinVisitDates.html b/docs/articles/checks/withinVisitDates.html index f3fd930e..dff31edf 100644 --- a/docs/articles/checks/withinVisitDates.html +++ b/docs/articles/checks/withinVisitDates.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • @@ -184,7 +190,7 @@

    withinVisitDates

    Clair Blacketer

    -

    2024-07-12

    +

    2024-10-24

    Source: vignettes/checks/withinVisitDates.Rmd diff --git a/docs/articles/index.html b/docs/articles/index.html index 0d30de15..5958ef1d 100644 --- a/docs/articles/index.html +++ b/docs/articles/index.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/authors.html b/docs/authors.html index 413f7d41..5e4321bb 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/index.html b/docs/index.html index bb851914..806e0a34 100644 --- a/docs/index.html +++ b/docs/index.html @@ -151,6 +151,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • diff --git a/docs/news/index.html b/docs/news/index.html index a5068eb0..05e0397b 100644 --- a/docs/news/index.html +++ b/docs/news/index.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index db534a4c..2aa6f2b6 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -34,7 +34,7 @@ articles: standardConceptRecordCompleteness: checks/standardConceptRecordCompleteness.html Thresholds: Thresholds.html withinVisitDates: checks/withinVisitDates.html -last_built: 2024-07-13T02:25Z +last_built: 2024-10-24T13:49Z urls: reference: https://ohdsi.github.io/DataQualityDashboard/reference article: https://ohdsi.github.io/DataQualityDashboard/articles diff --git a/docs/reference/convertJsonResultsFileCase.html b/docs/reference/convertJsonResultsFileCase.html index 5e74cad2..d506a628 100644 --- a/docs/reference/convertJsonResultsFileCase.html +++ b/docs/reference/convertJsonResultsFileCase.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/dot-applyNotApplicable.html b/docs/reference/dot-applyNotApplicable.html index 09ee42b8..3f3c00cd 100644 --- a/docs/reference/dot-applyNotApplicable.html +++ b/docs/reference/dot-applyNotApplicable.html @@ -131,12 +131,18 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog
  • Changelog
  • Changelog
  • Changelog diff --git a/docs/reference/dot-getCheckId.html b/docs/reference/dot-getCheckId.html index a179bf0b..2a02897f 100644 --- a/docs/reference/dot-getCheckId.html +++ b/docs/reference/dot-getCheckId.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/dot-hasNAchecks.html b/docs/reference/dot-hasNAchecks.html index bd38b694..0d87f5fa 100644 --- a/docs/reference/dot-hasNAchecks.html +++ b/docs/reference/dot-hasNAchecks.html @@ -131,12 +131,18 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog
  • Changelog diff --git a/docs/reference/dot-recordResult.html b/docs/reference/dot-recordResult.html index 65335d97..f144c0aa 100644 --- a/docs/reference/dot-recordResult.html +++ b/docs/reference/dot-recordResult.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/dot-runCheck.html b/docs/reference/dot-runCheck.html index 9a3af5e0..c6da7ca1 100644 --- a/docs/reference/dot-runCheck.html +++ b/docs/reference/dot-runCheck.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/dot-summarizeResults.html b/docs/reference/dot-summarizeResults.html index 4e6d8991..de59904b 100644 --- a/docs/reference/dot-summarizeResults.html +++ b/docs/reference/dot-summarizeResults.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/dot-writeResultsToCsv.html b/docs/reference/dot-writeResultsToCsv.html index 967f461d..bc2d9d71 100644 --- a/docs/reference/dot-writeResultsToCsv.html +++ b/docs/reference/dot-writeResultsToCsv.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/dot-writeResultsToJson.html b/docs/reference/dot-writeResultsToJson.html index 40b95795..a7a8a163 100644 --- a/docs/reference/dot-writeResultsToJson.html +++ b/docs/reference/dot-writeResultsToJson.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/dot-writeResultsToTable.html b/docs/reference/dot-writeResultsToTable.html index 56cb5037..bae7e51a 100644 --- a/docs/reference/dot-writeResultsToTable.html +++ b/docs/reference/dot-writeResultsToTable.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/executeDqChecks.html b/docs/reference/executeDqChecks.html index 99b2efd4..6d3cb41d 100644 --- a/docs/reference/executeDqChecks.html +++ b/docs/reference/executeDqChecks.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/index.html b/docs/reference/index.html index 216521ad..907618f8 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/listDqChecks.html b/docs/reference/listDqChecks.html index 0ccd3dc8..26ef4d0b 100644 --- a/docs/reference/listDqChecks.html +++ b/docs/reference/listDqChecks.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/reEvaluateThresholds.html b/docs/reference/reEvaluateThresholds.html index 23ab56c7..9c93e902 100644 --- a/docs/reference/reEvaluateThresholds.html +++ b/docs/reference/reEvaluateThresholds.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/viewDqDashboard.html b/docs/reference/viewDqDashboard.html index 82ed8c28..21c0397a 100644 --- a/docs/reference/viewDqDashboard.html +++ b/docs/reference/viewDqDashboard.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/writeDBResultsToJson.html b/docs/reference/writeDBResultsToJson.html index 509a9594..b721a73a 100644 --- a/docs/reference/writeDBResultsToJson.html +++ b/docs/reference/writeDBResultsToJson.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/writeJsonResultsToCsv.html b/docs/reference/writeJsonResultsToCsv.html index 28959148..1e6fa17d 100644 --- a/docs/reference/writeJsonResultsToCsv.html +++ b/docs/reference/writeJsonResultsToCsv.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/docs/reference/writeJsonResultsToTable.html b/docs/reference/writeJsonResultsToTable.html index 3c6f82cd..cf3649fc 100644 --- a/docs/reference/writeJsonResultsToTable.html +++ b/docs/reference/writeJsonResultsToTable.html @@ -131,6 +131,12 @@
  • measureConditionEraCompleteness
    • +
  • + plausibleUnitConceptIds +
    • +
  • + plausibleGenderUseDescendants +
  • Changelog diff --git a/vignettes/checkIndex.Rmd b/vignettes/checkIndex.Rmd index af436761..d46bf2ec 100644 --- a/vignettes/checkIndex.Rmd +++ b/vignettes/checkIndex.Rmd @@ -47,5 +47,5 @@ above to navigate to the check's documentation page.\ - [plausibleAfterBirth](checks/plausibleAfterBirth.html) - [plausibleBeforeDeath](checks/plausibleBeforeDeath.html) - [plausibleStartBeforeEnd](checks/plausibleStartBeforeEnd.html) -- plausibleGenderUseDescendants (PAGE UNDER CONSTRUCTION) -- plausibleUnitConceptIds (PAGE UNDER CONSTRUCTION) +- [plausibleGenderUseDescendants](checks/plausibleGenderUseDescendants.html) +- [plausibleUnitConceptIds](checks/plausibleUnitConceptIds.html) diff --git a/vignettes/checks/plausibleGenderUseDescendants.Rmd b/vignettes/checks/plausibleGenderUseDescendants.Rmd index 7fb6ca8b..d9d79400 100644 --- a/vignettes/checks/plausibleGenderUseDescendants.Rmd +++ b/vignettes/checks/plausibleGenderUseDescendants.Rmd @@ -1,6 +1,6 @@ --- title: "plausibleGender" -author: "" +author: "Katy Sadowski and Dmytry Dymshyts" date: "`r Sys.Date()`" output: html_document: @@ -14,33 +14,54 @@ output: **Context**: Validation\ **Category**: Plausibility\ **Subcategory**: Atemporal\ -**Severity**: +**Severity**: Characterization ✔ ## Description -For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGender). +For descendants of CONCEPT_ID conceptId (conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = plausibleGenderUseDescendants). ## Definition +This check will count the number of records for which the person's gender is implausible given the concept for the record. For a given gender-specific concept (e.g., prostate cancer) and its descendants, the check will identify records for which the associated person has an implausible gender (e.g., a female with prostate cancer). There are currently 4 instances of this check - female condition concepts; male condition concepts; female procedure concepts; and male procedure concepts. -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of rows with a gender-specific concept whose associated person has an implausible gender. +- *Denominator*: The number of rows with a gender-specific concept. +- *Related CDM Convention(s)*: https://ohdsi.github.io/Themis/populate_gender_concept_id.html +- *CDM Fields/Tables*: `CONDITION_OCURRENCE`, `PROCEDURE_OCCURRENCE` +- *Default Threshold Value*: 5% ## User Guidance +A failure of this check indicates one of the following scenarios: +- The person's gender is wrong +- The gender-specific concept is wrong +- The person changed genders and the concept was plausible at the time it was recorded ### Violated rows query ```sql - +SELECT + cdmTable.@cdmFieldName, + cdmTable.@sourceConceptIdField, -- x_source_concept_id for the table of interest (condition_occurrence or procedure_occurrence) + cdmTable.@sourceValueField, -- x_source_value for the table of interest + COUNT(*) +FROM @cdmDatabaseSchema.@cdmTableName cdmTable + JOIN @cdmDatabaseSchema.person p ON cdmTable.person_id = p.person_id + JOIN @vocabDatabaseSchema.concept_ancestor ca ON ca.descendant_concept_id = cdmTable.@cdmFieldName +WHERE ca.ancestor_concept_id IN (@conceptId) + AND p.gender_concept_id <> {@plausibleGenderUseDescendants == 'Male'} ? {8507} : {8532} +GROUP BY 1,2,3 ``` +The above query should help to identify if a mapping issue is the cause of the failure. If the source value and source concept ID are correctly mapped to a standard concept, then the issue may be that the person has the incorrect gender, or that the finding is a true data anomaly. Examples of true anomalies include: +- Occasional stray code (e.g., due to typo in EHR). This gives an idea how noisy the data is. +- Newborn codes recorded in the mother’s record (e.g., circumcision). +- Gender reassignment procedures (e.g., penectomy and prostatectomy in patients with acquired (female) gender). **NOTE** that this is technically a violation of the OMOP CDM specification, since the CDM actually requires that the `gender_concept_id` represents the person's sex at birth. For more information on this convention, see https://ohdsi.github.io/Themis/populate_gender_concept_id.html ### ETL Developers +Concept mapping issues must be fixed. Ensure as well that source codes are being correctly extracted from the source data. If the CDM accurately represents the source data, then remaining failures should be documented for users of the CDM. ### Data Users +Persons with implausible gender should not be included in analysis *unless* it can be confirmed with the data provider that these represent cases of gender reassignment, and your analysis does not assume that the `gender_concept_id` represents sex at birth. diff --git a/vignettes/checks/plausibleUnitConceptIds.Rmd b/vignettes/checks/plausibleUnitConceptIds.Rmd index 67f1a2bf..df4ba14a 100644 --- a/vignettes/checks/plausibleUnitConceptIds.Rmd +++ b/vignettes/checks/plausibleUnitConceptIds.Rmd @@ -1,6 +1,6 @@ --- title: "plausibleUnitConceptIds" -author: "" +author: "Katy Sadowski" date: "`r Sys.Date()`" output: html_document: @@ -14,7 +14,7 @@ output: **Context**: Verification\ **Category**: Plausibility\ **Subcategory**: Atemporal\ -**Severity**: +**Severity**: Characterization ✔ ## Description @@ -23,24 +23,43 @@ The number and percent of records for a given CONCEPT_ID @conceptId (@conceptNam ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of rows in the measurement table with a non-null `value_as_number` and a `unit_concept_id` that is not in the list of plausible unit concept ids. If the list of plausible unit concept ids includes -1, then a NULL `unit_concept_id` is accepted as a plausible unit. +- *Denominator*: The total number of rows in the measurement table with a non-null `value_as_number`. +- *Related CDM Convention(s)*: N/A +- *CDM Fields/Tables*: `MEASUREMENT` +- *Default Threshold Value*: 5% ## User Guidance +A failure of this check indicates one of the following: +- A measurement record has the incorrect unit +- A measurement record has no unit when it should have one +- A measurement record has a unit when it should not have one +- The list of plausible unit concept IDs in the threshold file is incomplete (please report this as a DataQualityDashboard bug!) + +The above issues could either be due to incorrect data in the source system or incorrect mapping of the unit concept IDs in the ETL process. ### Violated rows query ```sql - +SELECT + m.unit_concept_id, + m.unit_source_concept_id, + m.unit_source_value, + COUNT(*) +FROM @cdmDatabaseSchema.@cdmTableName m +WHERE m.@cdmFieldName = @conceptId + AND COALESCE (m.unit_concept_id, -1) NOT IN (@plausibleUnitConceptIds) -- '-1' stands for the cases when unit_concept_id is null + AND m.value_as_number IS NOT NULL +GROUP BY 1,2,3 ``` +Inspect the output of the violated rows query to identify the root cause of the issue. If the `unit_source_value` and/or `unit_source_concept_id` are populated, check them against the list of plausible unit concept IDs to understand if they should have been mapped to one of the plausible standard concepts. If the `unit_source_value` is NULL but the list of plausible unit concept IDs includes -1, then you may need to check your source data to undersand whether or not a unit is available in the source. ### ETL Developers +Ensure that all units available in the source data are being pulled into the CDM and mapped correctly to a standard concept ID. If a unit is available in the source and is being correctly populated & mapped in your ETL but is *not* present on the list of plausible unit concept IDs, you should verify whether or not the unit is actually plausible - you may need to consult a clinician to do so. If the unit is plausible for the given measurement, please report this as a DataQualityDashboard bug here: https://github.com/OHDSI/DataQualityDashboard/issues. If the unit is not plausible, do not change it! Instead, you should document the issue for users of the CDM and discuss with your data provider how to handle the data. ### Data Users +It is generally recommended not to use measurements with implausible units in analyses as it is impossible to determine whether the unit is wrong; the value is wrong; and/or the measurement code is wrong in the source data. If a measurement is missing a `unit_concept_id` due to an ETL issue, and the `unit_source_value` or `unit_source_concept_id` is available, you can utilize these values to perform your analysis. If `unit_source_value` and `unit_source_concept_id` are missing, you may consider consulting with your data provider as to if and when you may be able to infer what the missing unit should be. \ No newline at end of file From 447a321418b5ed38732b0f4c8aee5edd4da7fa9a Mon Sep 17 00:00:00 2001 From: Sam Hillman Date: Thu, 24 Oct 2024 09:58:12 -0400 Subject: [PATCH 02/12] Fix missing pull() and wrong launchLogViewer() file.path() argument Update the checksToExclude by changing the select() to a pull() to make sure we return the vector only rather than a column. Add explicit namespacing and call the library(dplyr) at the top of the commented code. Update the inspect logs code to correct the file.path() argument to include the cdmSourceName only in the .txt file name. --- extras/codeToRun.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extras/codeToRun.R b/extras/codeToRun.R index 596fffe1..662c2d9e 100644 --- a/extras/codeToRun.R +++ b/extras/codeToRun.R @@ -88,11 +88,12 @@ checkNames <- c() # Names can be found in inst/csv/OMOP_CDM_v5.3_Check_Descripti # want to EXCLUDE a pre-specified list of checks? run the following code: # +# library(dplyr) # checksToExclude <- c() # Names of check types to exclude from your DQD run # allChecks <- DataQualityDashboard::listDqChecks() # checkNames <- allChecks$checkDescriptions %>% # subset(!(checkName %in% checksToExclude)) %>% -# select(checkName) +# dplyr::pull(checkName) # which CDM tables to exclude? ------------------------------------ tablesToExclude <- c("CONCEPT", "VOCABULARY", "CONCEPT_ANCESTOR", "CONCEPT_RELATIONSHIP", "CONCEPT_CLASS", "CONCEPT_SYNONYM", "RELATIONSHIP", "DOMAIN") # list of CDM table names to skip evaluating checks against; by default DQD excludes the vocab tables @@ -118,7 +119,7 @@ DataQualityDashboard::executeDqChecks(connectionDetails = connectionDetails, checkNames = checkNames) # inspect logs ---------------------------------------------------------------------------- -ParallelLogger::launchLogViewer(logFileName = file.path(outputFolder, cdmSourceName, +ParallelLogger::launchLogViewer(logFileName = file.path(outputFolder, sprintf("log_DqDashboard_%s.txt", cdmSourceName))) # (OPTIONAL) if you want to write the JSON file to the results table separately ----------------------------- From 4f4c416a8ecabb5f660e5010b8981728d228600e Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Thu, 24 Oct 2024 09:59:29 -0400 Subject: [PATCH 03/12] fix GH Actions --- .github/workflows/R_CMD_check_Hades.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R_CMD_check_Hades.yml b/.github/workflows/R_CMD_check_Hades.yml index c7a63e86..f5b8ffc4 100644 --- a/.github/workflows/R_CMD_check_Hades.yml +++ b/.github/workflows/R_CMD_check_Hades.yml @@ -95,7 +95,7 @@ jobs: - name: Upload source package if: success() && runner.os == 'macOS' && github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: package_tarball path: check/*.tar.gz From 4a2e2aabe7abc7677946ac72e706b26426eb4b28 Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Thu, 24 Oct 2024 10:35:31 -0400 Subject: [PATCH 04/12] minor fixes --- .../checks/plausibleGenderUseDescendants.html | 15 +++++++-------- docs/articles/checks/plausibleUnitConceptIds.html | 12 ++++++------ docs/articles/index.html | 2 +- docs/pkgdown.yml | 2 +- .../checks/plausibleGenderUseDescendants.Rmd | 6 +++--- vignettes/checks/plausibleUnitConceptIds.Rmd | 8 ++++---- 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/docs/articles/checks/plausibleGenderUseDescendants.html b/docs/articles/checks/plausibleGenderUseDescendants.html index 985ffb09..82fae2b0 100644 --- a/docs/articles/checks/plausibleGenderUseDescendants.html +++ b/docs/articles/checks/plausibleGenderUseDescendants.html @@ -5,13 +5,13 @@ -plausibleGender • DataQualityDashboard +plausibleGenderUseDescendants • DataQualityDashboard - +