Merge branch 'develop' into main

.github/workflows/deployment_production.yml

@@ -33,5 +33,5 @@ jobs:
           --set-json='images.backend="ghcr.io/${{ github.repository }}/backend:${{ github.event.pull_request.head.sha || github.sha }}"' \
           --set-json='images.frontend="ghcr.io/${{ github.repository }}/frontend:${{ github.event.pull_request.head.sha || github.sha }}"'  \
           --set-json='images.prediction="gitlab.zbmed.de:5050/km/thesis/pierre-ba/backend:a7d3254ee488a236dade40ecedc646861aff9dbe"'  \
-          --set-json='ingress.dns="mda.main.prod.km.k8s.zbmed.de"'  \
+          --set-json='ingress.dns="mda-prod.prod.km.k8s.zbmed.de"'  \
           mda-deployment/metadata-annotation

.gitignore

@@ -888,3 +888,4 @@ frontend/yarn.lock
 
 backend/instruments.db
 instruments.db
+frontend/.npmrc

backend/restapi/__init__.py

@@ -228,7 +228,33 @@ def get_columns():
         except OSError:
             pass
 
-        return jsonify(list(df.columns.values))
+        return(jsonify(list(df.columns.values)))
+
+    @app.route('/api/instrument/columnCheck', methods=["POST"])
+    def column_check():
+        file_to_import = request.files.get("file", None)
+
+        if not os.path.exists(INSTRUMENTS + "/tmp"):
+            os.makedirs(INSTRUMENTS + "/tmp")
+
+        file_to_import.save(INSTRUMENTS + "/tmp/" + file_to_import.filename)
+
+        if file_to_import.filename.split(".")[-1] == "xlsx":
+            df = pd.read_excel(INSTRUMENTS + "/tmp/" + file_to_import.filename)
+        if file_to_import.filename.split(".")[-1] == "csv":
+            df = pd.read_csv(INSTRUMENTS + "/tmp/" + file_to_import.filename)
+
+        required_opal_columns = ["index", "table", "name", "valueType", "unit", "label:en", "label:la", "label:de",
+                                 "alias"]
+        actual_columns = df.columns.values
+        missing_columns = [i for i in required_opal_columns if i not in actual_columns]
+
+        try:
+            os.remove(INSTRUMENTS + "/tmp/" + file_to_import.filename)
+        except OSError:
+            pass
+
+        return(jsonify(missing_columns))
 
     @app.route('/api/instrument', methods=["POST"])
     def import_instrument():
@@ -391,16 +417,25 @@ def auto_annotation():
             query_code = db.session.query(Code).filter_by(instrument_name=projectId) \
                 .filter_by(code_linkId=dic["linkId"]).all()
             codes = [c.as_dict() for c in query_code]
-            for element in codes:
-                if prediction_iri == element["code"]:
-                    return jsonify('isInDB')
-            if not isInDB:
+            if(len(codes) > 0):
+                for element in codes:
+                    if prediction_iri == element["code"]:
+                        isInDB = True
+                    if not isInDB:
+                        stmt = (
+                            insert(Code).
+                            values(code_linkId=dic["linkId"], code=prediction_iri, instrument_name=projectId)
+                        )
+                        session.execute(stmt)
+                        session.commit()
+            else:
                 stmt = (
                     insert(Code).
                     values(code_linkId=dic["linkId"], code=prediction_iri, instrument_name=projectId)
                 )
                 session.execute(stmt)
                 session.commit()
+
         return
 
     @app.route('/api/annotations', methods=['DELETE'])

backend/restapi/services/exporter/export_maelstrom.py

@@ -1,7 +1,7 @@
-import os
+from urllib.parse import quote
+
 import pandas as pd
 import requests
-from urllib.parse import quote
 
 
 def get_codes_for_linkId(linkId, codes):
@@ -11,9 +11,42 @@ def get_codes_for_linkId(linkId, codes):
             codelist.append(ele.code)
     return codelist
 
+
+def get_all_domains():
+    # TODO code does only returns 14 of 18 domains
+    # TODO add Maelstrom Additionals
+    url = "https://semanticlookup.zbmed.de/ols/api/ontologies/maelstrom/terms?size=500"
+    response = requests.get(url).json()
+    all_domains = ["Mlstr_area::" + domain["synonyms"][0].replace(' ', '_') for domain in response["_embedded"]["terms"]
+                   if (domain["synonyms"] is not None and domain["is_root"])]
+    return [
+        'Mlstr_area::Symptoms_signs',
+        'Mlstr_area::Non_pharmacological_interventions',
+        'Mlstr_area::Health_community_care_utilization',
+        'Mlstr_area::Sociodemographic_economic_characteristics',
+        'Mlstr_area::Social_environment',
+        'Mlstr_area::Medication_supplements',
+        'Mlstr_area::Preschool_school_work',
+        'Mlstr_area::Life_events_plans_beliefs',
+        'Mlstr_area::Lifestyle_behaviours',
+        'Mlstr_area::Cognitive_psychological_measures',
+        'Mlstr_area::End_of_life',
+        'Mlstr_area::Physical_measures',
+        'Mlstr_area::Health_status_functional_limitations',
+        'Mlstr_area::Reproduction',
+        'Mlstr_area::Diseases',
+        'Mlstr_area::Laboratory_measures',
+        'Mlstr_area::Physical_environment',
+        'Mlstr_area::Administrative_information',
+    ]
+
+
 def export_maelstrom_annotations_opal(df, instrument, questions, codes):
-    # df = pd.read_excel(os.path.join(instruments, instrument[0].original_name))
-    # df = df.reset_index()
+    all_domains = get_all_domains()
+
+    # add all domains to df
+    df = pd.concat([df, pd.DataFrame(columns=all_domains)])
+
     for index, row in df.iterrows():
         annotation_col = instrument[0].annotation_column
         if isinstance(row[annotation_col], str):
@@ -29,11 +62,19 @@ def export_maelstrom_annotations_opal(df, instrument, questions, codes):
                         response = requests.get(url).json()
 
                         if response["_embedded"]["terms"][0]["ontology_name"] != "maelstrom":
-                            return("Not a maelstrom concept")
+                            return ("Not a maelstrom concept")
+
+                        if (response["_embedded"]["terms"][0]["synonyms"] is None):
+                            label = response["_embedded"]["terms"][0]["label"]
+                        else:
+                            label = response["_embedded"]["terms"][0]["synonyms"][0].replace(' ', '_')
 
-                        label = response["_embedded"]["terms"][0]["label"]
-                        response_parent = requests.get("https://semanticlookup.zbmed.de/ols/api/ontologies/maelstrom/terms/" + iri_encoded + "/parents").json()
-                        label_parent = response_parent["_embedded"]["terms"][0]["label"]
+                        response_parent = requests.get(
+                            "https://semanticlookup.zbmed.de/ols/api/ontologies/maelstrom/terms/" + iri_encoded + "/parents").json()
+                        if(response_parent["_embedded"]["terms"][0]["synonyms"] is None):
+                            label_parent = response_parent["_embedded"]["terms"][0]["label"].replace(' ', '_')
+                        else:
+                            label_parent = response_parent["_embedded"]["terms"][0]["synonyms"][0].replace(' ', '_')
 
                         maelstrom_prefix = "Mlstr_area::"
 
@@ -42,8 +83,8 @@ def export_maelstrom_annotations_opal(df, instrument, questions, codes):
     df = df[df.filter(regex='^(?!Unnamed)').columns]
     return df
 
-def export_maelstrom_annotations_opal_only_annotations(questions, codes):
 
+def export_maelstrom_annotations_opal_only_annotations(questions, codes):
     df = pd.DataFrame(columns=['label'])
 
     for q in questions:
@@ -57,10 +98,11 @@ def export_maelstrom_annotations_opal_only_annotations(questions, codes):
             response = requests.get(url).json()
 
             if response["_embedded"]["terms"][0]["ontology_name"] != "maelstrom":
-                return("Not a maelstrom concept")
+                return ("Not a maelstrom concept")
 
             label = response["_embedded"]["terms"][0]["label"]
-            response_parent = requests.get("https://semanticlookup.zbmed.de/ols/api/ontologies/maelstrom/terms/" + iri_encoded + "/parents").json()
+            response_parent = requests.get(
+                "https://semanticlookup.zbmed.de/ols/api/ontologies/maelstrom/terms/" + iri_encoded + "/parents").json()
             label_parent = response_parent["_embedded"]["terms"][0]["label"]
 
             maelstrom_prefix = "Mlstr_area::"
@@ -72,4 +114,4 @@ def export_maelstrom_annotations_opal_only_annotations(questions, codes):
                     if q.text == row['label']:
                         df.at[index, maelstrom_prefix + label_parent] = label
 
-    return df
+    return df