Updating TCR ERD (#2281)

* Updating TCR ERD

* setting up junction tables, updating variable/column names

* finishing updates to db

* fixing ersd junction code, updating function names

* updating documentation, doc string

* fixing typo

* updating tables created

* updating documentation

* updating with latest erd

* adding VSAC FHIR API code to capture versions

* optimizing code

* updating comments, checking integration tests

* trying different version of testcontainers

* changing type_id to type

* rolling back req change

* we do need to change dev-reqs?

* updating to db

* [pre-commit.ci] auto fixes from pre-commit hooks

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

containers/trigger-code-reference/.gitignore

@@ -0,0 +1 @@
+.env

containers/trigger-code-reference/app/main.py

@@ -11,8 +11,8 @@
 from app.utils import _stamp_resource_with_code_extension
 from app.utils import find_conditions
 from app.utils import get_clean_snomed_code
-from app.utils import get_clinical_services_dict
-from app.utils import get_clinical_services_list
+from app.utils import get_concepts_dict
+from app.utils import get_concepts_list
 from app.utils import read_json_from_assets
 
 RESOURCE_TO_SERVICE_TYPES = {
@@ -78,8 +78,8 @@ async def stamp_condition_extensions(
     conditions = find_conditions(input.bundle)
 
     for cond in conditions:
-        cond_list = get_clinical_services_list([cond])
-        cond_dict = get_clinical_services_dict(cond_list)
+        cond_list = get_concepts_list([cond])
+        cond_dict = get_concepts_dict(cond_list)
         stamp_codes_to_service_codes[cond] = cond_dict
 
     bundle_entries = input.bundle.get("entry", [])
@@ -135,7 +135,7 @@ async def stamp_condition_extensions(
 @app.get("/get-value-sets", status_code=200, responses=get_value_sets_response_examples)
 async def get_value_sets_for_condition(
     condition_code: Annotated[str, Query(examples=get_value_sets_request_examples)],
-    filter_clinical_services: Annotated[
+    filter_concepts: Annotated[
         str, Query(examples=get_value_sets_request_examples)
     ] = None,
 ) -> Response:
@@ -145,9 +145,9 @@ async def get_value_sets_for_condition(
 
     :param condition_code: A query param supplied as a string representing a
       single SNOMED condition code.
-    :param filter_clinical_services: (Optional) A comma-separated string of
-      clinical service types (defined by the abbreviation codes above) to
-      keep. By default, all (currently) 6 clinical service types are
+    :param filter_concepts: (Optional) A comma-separated string of
+      value set types (defined by the abbreviation codes above) to
+      keep. By default, all (currently) 6 value set types are
       returned; use this parameter to return only types of interest.
     :return: An HTTP Response containing the value sets of the queried code.
     """
@@ -158,10 +158,8 @@ async def get_value_sets_for_condition(
         )
     else:
         clean_snomed_code = get_clean_snomed_code(condition_code)
-        clinical_services_list = get_clinical_services_list(clean_snomed_code)
-        values = get_clinical_services_dict(
-            clinical_services_list, filter_clinical_services
-        )
+        concepts_list = get_concepts_list(clean_snomed_code)
+        values = get_concepts_dict(concepts_list, filter_concepts)
     return values
 
 

containers/trigger-code-reference/app/utils.py

@@ -46,93 +46,92 @@ def get_clean_snomed_code(snomed_code: Union[list, str, int, float]) -> list:
     return clean_snomed_code
 
 
-def get_clinical_services_list(snomed_code: list) -> List[tuple]:
+def get_concepts_list(snomed_code: list) -> List[tuple]:
     """
     This will take a SNOMED code and runs a SQL query joins condition code,
     joins it to value sets, then uses the value set ids to get the
-    clinical service type, clinical service codes, and clinical service system
-    from the eRSD database grouped by clinical service type and system.
+    value set type, concept codes, and concept system
+    from the eRSD database grouped by value set type and system.
 
     :param snomed_code: SNOMED code to check
-    :return: A list of tuples with clinical service type, a delimited-string of
+    :return: A list of tuples with value set type, a delimited-string of
     the relevant codes and code systems as objects within.
     """
     sql_query = """
     SELECT
-        vs.clinical_service_type_id AS clinical_service_type,
+        vs.type AS valueset_type,
         GROUP_CONCAT(cs.code, '|') AS codes,
         cs.code_system AS system
     FROM
         conditions c
-    JOIN
-        value_sets vs ON c.value_set_id = vs.id
-    JOIN
-        clinical_services cs ON cs.value_set_id = vs.id
+    LEFT JOIN
+        condition_to_valueset cv ON c.id = cv.condition_id
+    LEFT JOIN
+        valuesets vs ON cv.valueset_id = vs.id
+    LEFT JOIN
+        valueset_to_concept vc ON vs.id = vc.valueset_id
+    LEFT JOIN
+        concepts cs ON vc.concept_id = cs.id
     WHERE
         c.id = ?
     GROUP BY
-        vs.clinical_service_type_id, cs.code_system
+        vs.type, cs.code_system
     """
-
     # Connect to the SQLite database, execute sql query, then close
     try:
         with sqlite3.connect("seed-scripts/ersd.db") as conn:
             cursor = conn.cursor()
             code = get_clean_snomed_code(snomed_code)
             cursor.execute(sql_query, code)
-            clinical_services_list = cursor.fetchall()
+            concept_list = cursor.fetchall()
             # We know it's not an actual error because we didn't get kicked to
             # except, so just return the lack of results
-            if not clinical_services_list:
+            if not concept_list:
                 return []
-        return clinical_services_list
+        return concept_list
     except sqlite3.Error as e:
         return {"error": f"An SQL error occurred: {str(e)}"}
 
 
-def get_clinical_services_dict(
-    clinical_services_list: List[tuple],
-    filter_clinical_services: Union[str, list] = None,
+def get_concepts_dict(
+    concept_list: List[tuple],
+    filter_concept_list: Union[str, list] = None,
 ) -> dict:
     """
     This function parses a list of tuples containing data on clinical codes
     into a dictionary for use in the /get-value-sets API endpoint.
 
-    There is an optional parameter to return select clinical service type(s)
+    There is an optional parameter to return select value set type(s)
     specified as either a string or a list.
 
-    :param clinical_services_list: A list of tuples with clinical service type,
+    :param concept_list: A list of tuples with value set type,
     a delimited-string of relevant codes and code systems as objects within.
-    :param filter_clinical_services: (Optional) List of clinical service types
-    specified to keep. By default, all (currently) 6 clinical service types are
+    :param filter_concept_list: (Optional) List of value set types
+    specified to keep. By default, all (currently) 6 value set types are
     returned; use this parameter to return only types of interest.
-    :return: A nested dictionary with clinical service type as the key, a list
+    :return: A nested dictionary with value set type as the key, a list
     of the relevant codes and code systems as objects within.
     """
     # Convert to the final structured format
-    clinical_service_dict = {}
-    for clinical_service_type, codes_string, system in clinical_services_list:
-        # If clinical_service_type is not yet in the dictionary, initialize
-        if clinical_service_type not in clinical_service_dict:
-            clinical_service_dict[clinical_service_type] = []
+    concept_dict = {}
+    for concept_type, codes_string, system in concept_list:
+        # If concept_type is not yet in the dictionary, initialize
+        if concept_type not in concept_dict:
+            concept_dict[concept_type] = []
         # Append a new entry with the codes and their system
-        clinical_service_dict[clinical_service_type].append(
+        concept_dict[concept_type].append(
             {"codes": codes_string.split("|"), "system": system}
         )
 
-    # Optional: Remove clinical service types not in specified list if provided
-    if filter_clinical_services:
-        clinical_services = convert_inputs_to_list(filter_clinical_services)
+    # Optional: Remove value set types not in specified list if provided
+    if filter_concept_list:
+        concepts = convert_inputs_to_list(filter_concept_list)
         # Create a list of types to remove
-        remove_list = [
-            type
-            for type in clinical_service_dict.keys()
-            if type not in clinical_services
-        ]
+        remove_list = [type for type in concept_dict.keys() if type not in concepts]
         # Remove the types
         for type in remove_list:
-            clinical_service_dict.pop(type, None)
-    return clinical_service_dict
+            concept_dict.pop(type, None)
+    return concept_dict
 
 
 def _find_codes_by_resource_type(resource: dict) -> List[str]:

containers/trigger-code-reference/assets/sample_get_value_sets_requests.json

@@ -7,11 +7,11 @@
     }
   },
   "Filtered value set retrieval": {
-    "summary": "Retrieves and filteres COVID-related value sets using a clinical services filter",
+    "summary": "Retrieves and filters COVID-related value sets using a concept type filter",
     "description": "This example queries the SQLite database for value sets related to condition code 840539006; we're interested only in lab-related services for this code, so we filter out diagnostics.",
     "value": {
       "condition_code": "840539006",
-      "filter_clinical_services": "lrtc,ostc"
+      "filter_concepts": "lrtc,ostc"
     }
   }
 }

containers/trigger-code-reference/dev-requirements.txt

@@ -1,6 +1,6 @@
 pytest
 httpx
-testcontainers[compose]==3.7.1
+testcontainers[compose]
 mammoth==1.8.0
 beautifulsoup4==4.12.3
 lxml==5.2.2

containers/trigger-code-reference/seed-scripts/config/ersd.json

@@ -1,5 +1,5 @@
 {
-  "value_set_type": {
+  "valueset_types": {
     "fhir_path": "entry.resource.where(resourceType='ValueSet' and url.contains('http://ersd.aimsplatform.org/fhir/ValueSet/')",
     "data_type": "string",
     "nullable": false,
@@ -9,19 +9,19 @@
         "data_type": "string",
         "nullable": false
       },
-      "clinical_service_type": {
+      "concept_type": {
         "fhir_path": "title",
         "data_type": "string",
         "nullable": false
       }
     }
   },
-  "clinical_services": {
+  "concepts": {
     "fhir_path": "entry.resource.where(resourceType='ValueSet' and url.contains('http://cts.nlm.nih.gov/fhir/ValueSet/')",
     "data_type": "string",
     "nullable": false,
     "secondary_schema": {
-      "value_set_id": {
+      "valueset_id": {
         "fhir_path": "id",
         "data_type": "string",
         "nullable": false
@@ -48,17 +48,17 @@
       }
     }
   },
-  "value_sets": {
+  "valuesets": {
     "fhir_path": "entry.resource.where(resourceType='ValueSet' and url.contains('http://ersd.aimsplatform.org/fhir/ValueSet/')",
     "data_type": "string",
     "nullable": false,
     "secondary_schema": {
-      "clinical_service_type_id": {
+      "concept_type_id": {
         "fhir_path": "id",
         "data_type": "string",
         "nullable": false
       },
-      "version": {
+      "ersd_version": {
         "fhir_path": "version",
         "data_type": "string",
         "nullable": false

containers/trigger-code-reference/seed-scripts/migrations/V02_01__create_tables_add_indexes.sql

@@ -0,0 +1,59 @@
+CREATE TABLE IF NOT EXISTS valuesets (
+    id TEXT PRIMARY KEY,
+    oid TEXT,
+    version TEXT,
+    name TEXT,
+    author TEXT,
+    type TEXT
+);
+
+CREATE TABLE IF NOT EXISTS conditions (
+    id TEXT PRIMARY KEY,
+    system TEXT,
+    name TEXT,
+    version TEXT
+);
+
+CREATE TABLE IF NOT EXISTS concepts (
+    id TEXT PRIMARY KEY,
+    code TEXT,
+    code_system TEXT,
+    display TEXT,
+    version TEXT
+);
+
+CREATE TABLE IF NOT EXISTS condition_to_valueset (
+    id TEXT PRIMARY KEY,
+    condition_id TEXT,
+    valueset_id TEXT,
+    source TEXT,
+    FOREIGN KEY (condition_id) REFERENCES conditions(id),
+    FOREIGN KEY (valueset_id) REFERENCES valuesets(id)
+);
+
+CREATE TABLE IF NOT EXISTS valueset_to_concept (
+    id TEXT PRIMARY KEY,
+    valueset_id TEXT,
+    concept_id TEXT,
+    FOREIGN KEY (valueset_id) REFERENCES valuesets(id),
+    FOREIGN KEY (concept_id) REFERENCES concepts(id)
+);
+
+
+-- add indexes to increase performance
+-- conditions
+CREATE INDEX IF NOT EXISTS "idx_conditions_id" ON conditions(id);
+
+-- valuesets
+CREATE INDEX IF NOT EXISTS "idx_valuesets_id" ON valuesets(id);
+
+-- concepts
+CREATE INDEX IF NOT EXISTS "idx_concepts_id" ON concepts(id);
+
+-- valueset_to_concept indexes
+CREATE INDEX IF NOT EXISTS "idx_valueset_to_concept_valueset_id" ON valueset_to_concept(valueset_id);
+CREATE INDEX IF NOT EXISTS "idx_valueset_to_concept_concept_id" ON valueset_to_concept(concept_id);
+
+-- condition_to_valueset indexes
+CREATE INDEX IF NOT EXISTS "idx_condition_to_valueset_condition_id" ON condition_to_valueset(condition_id);
+CREATE INDEX IF NOT EXISTS "idx_condition_to_valueset_valueset_id" ON condition_to_valueset(valueset_id);