Merge pull request #142 from CogStack/cui-viewer-page

A simple CUI Viewer Page and CUI filter selector

webapp/api/api/medcat_utils.py

@@ -0,0 +1,68 @@
+import logging
+from collections import defaultdict
+from typing import List
+
+from medcat.cdb import CDB
+
+
+logger = logging.getLogger(__name__)
+
+
+def ch2pt_from_pt2ch(cdb: CDB):
+    ch2pt = defaultdict(list)
+    for k, vals in cdb.addl_info['pt2ch'].items():
+        for v in vals:
+            ch2pt[v].append(k)
+    return ch2pt
+
+
+def get_all_ch(parent_cui: str, cdb: CDB):
+    all_ch = [parent_cui]
+    for cui in cdb.addl_info['pt2ch'].get(parent_cui, []):
+        cui_chs = get_all_ch(cui, cdb)
+        all_ch += cui_chs
+    return dedupe_preserve_order(all_ch)
+
+
+def dedupe_preserve_order(items: List[str]) -> List[str]:
+    seen = set()
+    deduped_list = []
+    for item in items:
+        if item not in seen:
+            seen.add(item)
+            deduped_list.append(item)
+    return deduped_list
+
+
+def snomed_ct_concept_path(cui: str, cdb: CDB):
+    try:
+        top_level_parent_node = '138875005'
+
+        def find_parents(cui, cuis2nodes, child_node=None):
+            parents = list(cdb.addl_info['ch2pt'][cui])
+            all_links = []
+            if cui not in cuis2nodes:
+                curr_node = {'cui': cui, 'pretty_name': cdb.cui2preferred_name[cui]}
+                if child_node:
+                    curr_node['children'] = [child_node]
+                cuis2nodes[cui] = curr_node
+                if len(parents) > 0:
+                    all_links += find_parents(parents[0], cuis2nodes, child_node=curr_node)
+                    for p in parents[1:]:
+                        links = find_parents(p, cuis2nodes)
+                        all_links += [{'parent': p, 'child': cui}] + links
+            else:
+                if child_node:
+                    if 'children' not in cuis2nodes[cui]:
+                        cuis2nodes[cui]['children'] = []
+                    cuis2nodes[cui]['children'].append(child_node)
+            return all_links
+        cuis2nodes = dict()
+        all_links = find_parents(cui, cuis2nodes)
+        return {
+            'node_path': cuis2nodes[top_level_parent_node],
+            'links': all_links
+        }
+    except KeyError as e:
+        logger.warning(f'Cannot find path concept path:{e}')
+        return []

webapp/api/api/utils.py

@@ -325,6 +325,14 @@ def get_medcat(CDB_MAP, VOCAB_MAP, CAT_MAP, project):
     return cat
 
 
+def get_cached_cdb(cdb_id: str, CDB_MAP: Dict[str, CDB]) -> CDB:
+    if cdb_id not in CDB_MAP:
+        cdb_obj = ConceptDB.objects.get(id=cdb_id)
+        cdb = CDB.load(cdb_obj.cdb_file.path)
+        CDB_MAP[cdb_id] = cdb
+    return CDB_MAP[cdb_id]
+
+
 @receiver(post_save, sender=ProjectAnnotateEntities)
 def save_project_anno(sender, instance, **kwargs):
     if instance.cuis_file:

webapp/api/api/views.py

@@ -3,23 +3,25 @@
 import traceback
 from tempfile import NamedTemporaryFile
 
-from django.http import HttpResponseBadRequest, HttpResponseServerError
+from django.http import HttpResponseBadRequest, HttpResponseServerError, HttpResponse
 from django.shortcuts import render
 from django_filters import rest_framework as drf
+from medcat.cdb import CDB
 from medcat.utils.helpers import tkns_from_doc
 from rest_framework import viewsets
 from rest_framework.decorators import api_view
 from rest_framework.response import Response
 
-
+from core.settings import MEDIA_ROOT
 from .admin import download_projects_with_text, download_projects_without_text, \
     import_concepts_from_cdb, upload_projects_export, retrieve_project_data
+from .medcat_utils import ch2pt_from_pt2ch, get_all_ch, dedupe_preserve_order, snomed_ct_concept_path
 from .metrics import ProjectMetrics
 from .permissions import *
 from .serializers import *
 from .solr_utils import collections_available, search_collection, ensure_concept_searchable
-from .utils import get_cached_medcat, clear_cached_medcat
-from .utils import get_medcat, add_annotations, remove_annotations, train_medcat, create_annotation
+from .utils import get_cached_medcat, clear_cached_medcat, get_medcat, get_cached_cdb, \
+    add_annotations, remove_annotations, train_medcat, create_annotation
 
 # For local testing, put envs
 """
@@ -626,6 +628,12 @@ def upload_deployment(request):
     return Response("successfully uploaded", 200)
 
 
+@api_view(http_method_names=['GET'])
+def cache_model(_, cdb_id):
+    get_cached_cdb(cdb_id, CDB_MAP)
+    return Response("success", 200)
+
+
 @api_view(http_method_names=['GET'])
 def model_loaded(_):
     return Response({p.id: get_cached_medcat(CAT_MAP, p) is not None
@@ -656,4 +664,80 @@ def metrics(request):
     return Response({'results': report_output})
 
 
+@api_view(http_method_names=['GET'])
+def cdb_cui_children(request, cdb_id):
+    parent_cui = request.GET.get('parent_cui')
+    cdb = get_cached_cdb(cdb_id, CDB_MAP)
+
+    # root SNOMED CT code: 138875005
+    # root UMLS code: CUI:
+    # root level ICD term:
+    # root level OPCS term:
+
+    if cdb.addl_info.get('pt2ch') is None:
+        return HttpResponseBadRequest('Requested MedCAT CDB model does not include parent2child metadata to'
+                                      ' explore a concept hierarchy')
+
+    # currently assumes this is using the SNOMED CT terminology
+    try:
+        root_term = {'cui': '138875005', 'pretty_name': cdb.cui2preferred_name['138875005']}
+        if parent_cui is None:
+            return Response({'results': [root_term]})
+        else:
+            child_concepts = [{'cui': cui, 'pretty_name': cdb.cui2preferred_name[cui]}
+                              for cui in cdb.addl_info.get('pt2ch')[parent_cui]]
+            return Response({'results': child_concepts})
+    except KeyError:
+        return Response({'results': []})
+
+
+@api_view(http_method_names=['GET'])
+def cdb_concept_path(request):
+    cdb_id = int(request.GET.get('cdb_id'))
+    cdb = get_cached_cdb(cdb_id, CDB_MAP)
+    if not cdb.addl_info.get('ch2pt'):
+        cdb.addl_info['ch2pt'] = ch2pt_from_pt2ch(cdb)
+    cui = request.GET.get('cui')
+    # Again only SNOMED CT is supported
+    # 'cui': '138875005',
+    result = snomed_ct_concept_path(cui, cdb)
+    return Response({'results': result})
+
+
+@api_view(http_method_names=['POST'])
+def generate_concept_filter_flat_json(request):
+    cuis = request.data.get('cuis')
+    cdb_id = request.data.get('cdb_id')
+    excluded_nodes = request.data.get('excluded_nodes', [])
+    if cuis is not None and cdb_id is not None:
+        cdb = get_cached_cdb(cdb_id, CDB_MAP)
+        # get all children from 'parent' concepts above.
+        final_filter = []
+        for cui in cuis:
+            ch_nodes = get_all_ch(cui, cdb)
+            final_filter += [n for n in ch_nodes if n not in excluded_nodes]
+        final_filter = dedupe_preserve_order(final_filter)
+        filter_json = json.dumps(final_filter)
+        response = HttpResponse(filter_json, content_type='application/json')
+        response['Content-Disposition'] = 'attachment; filename=filter.json'
+        return response
+    return HttpResponseBadRequest('Missing either cuis or cdb_id param. Cannot generate filter.')
+
 
+@api_view(http_method_names=['POST'])
+def generate_concept_filter(request):
+    cuis = request.data.get('cuis')
+    cdb_id = request.data.get('cdb_id')
+    if cuis is not None and cdb_id is not None:
+        cdb = get_cached_cdb(cdb_id, CDB_MAP)
+        # get all children from 'parent' concepts above.
+        final_filter = {}
+        for cui in cuis:
+            final_filter[cui] = [{'cui': c, 'pretty_name': cdb.cui2preferred_name[c]} for c in get_all_ch(cui, cdb)
+                                 if c in cdb.cui2preferred_name and c != cui]
+        resp = {'filter_len': sum(len(f) for f in final_filter.values()) + len(final_filter.keys())}
+        if resp['filter_len'] < 10000:
+            # only send across concept filters that are small enough to render
+            resp['filter'] = final_filter
+        return Response(resp)
+    return HttpResponseBadRequest('Missing either cuis or cdb_id param. Cannot generate filter.')

webapp/api/core/urls.py

@@ -59,8 +59,12 @@
     path('api/version/', api.views.version),
     path('api/concept-db-search-index-created/', api.views.concept_search_index_available),
     path('api/model-loaded/', api.views.model_loaded),
-    path('api/cache-model/<int:p_id>/', api.views.cache_model),
+    path('api/cache_model/<int:cdb_id>', api.views.cache_model),
     path('api/upload-deployment/', api.views.upload_deployment),
+    path('api/model-concept-children/<int:cdb_id>/', api.views.cdb_cui_children),
     path('api/metrics/', api.views.metrics),
+    path('api/concept-path/', api.views.cdb_concept_path),
+    path('api/generate-concept-filter-json/', api.views.generate_concept_filter_flat_json),
+    path('api/generate-concept-filter/', api.views.generate_concept_filter),
     re_path('^.*$', api.views.index, name='index'),  # Match everything else to home
 ]